01HP3NZXBJ2Q39QDHRP708ENYW: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HP3P09GPGCG1BCDMV1B2P7SK

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-08T06:36:26.351Zjob dependencies complete; ready to run (waiting for 15 m 44 s)
22024-02-08T06:37:24.731Zjob assigned to worker 01HP3PX8HXCDX8307FHJR5SHTQ (queued for 58 s)
32024-02-08T06:37:25.769Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-08T06:37:45.970Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-08T06:37:45.971Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-08T06:38:01.879Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-08T06:38:01.879Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-08T06:38:19.389Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-08T06:38:19.389Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-08T06:38:36.091Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-08T06:38:36.097Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-08T06:38:36.113Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-08T06:38:36.113Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-08T06:38:36.132Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-08T06:38:36.132Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-08T06:38:36.151Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-08T06:38:36.151Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-08T06:38:36.696Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-08T06:38:36.696Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-08T06:38:36.729Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-08T06:38:36.729Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-08T06:38:36.761Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-08T06:38:36.761Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-08T06:38:36.781Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-08T06:38:36.803Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-08T06:38:55.264Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-08T06:38:55.264Zstarting task 0: "setup"
282024-02-08T06:38:55.572Z++ uname -s
292024-02-08T06:38:55.576Z+ kern=SunOS
302024-02-08T06:38:55.576Z+ case "$kern" in
312024-02-08T06:38:55.576Z+ groupadd -g 12345 build
322024-02-08T06:38:55.580Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-08T06:38:57.588Z+ zfs create -o mountpoint=/work rpool/work
342024-02-08T06:38:57.866Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-08T06:38:57.883Z+ home_fs=zfs
362024-02-08T06:38:57.883Z+ [[ zfs == autofs ]]
372024-02-08T06:38:57.883Z+ mkdir -p /home/build
382024-02-08T06:38:57.884Z+ chown build:build /home/build /work
392024-02-08T06:38:58.864Z+ chmod 0700 /home/build /work
402024-02-08T06:38:59.314Zprocess exited: duration 3605 ms, exit code 0
 
412024-02-08T06:38:59.322Zstarting task 1: "authentication"
422024-02-08T06:38:59.342Zprocess exited: duration 20 ms, exit code 0
 
432024-02-08T06:38:59.348Zstarting task 2: "build"
442024-02-08T06:38:59.356Z+ banner cores
452024-02-08T06:38:59.390Z
462024-02-08T06:38:59.390Z #### #### ##### ###### ####
472024-02-08T06:38:59.390Z # # # # # # # #
482024-02-08T06:38:59.390Z # # # # # ##### ####
492024-02-08T06:38:59.390Z # # # ##### # #
502024-02-08T06:38:59.390Z # # # # # # # # #
512024-02-08T06:38:59.391Z #### #### # # ###### ####
522024-02-08T06:38:59.391Z
532024-02-08T06:38:59.394Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-08T06:38:59.400Z+ echo 'input bins dir contains:'
552024-02-08T06:38:59.403Zinput bins dir contains:
562024-02-08T06:38:59.403Z+ ls -ltr /input/build/work/bins
572024-02-08T06:38:59.403Ztotal 763503
582024-02-08T06:38:59.403Z-rw-r--r-- 1 root root 122132441 Feb 8 06:37 crucible-downstairs.gz
592024-02-08T06:38:59.403Z-rw-r--r-- 1 root root 91406059 Feb 8 06:38 crucible-hammer.gz
602024-02-08T06:38:59.403Z-rw-r--r-- 1 root root 105100057 Feb 8 06:38 crutest.gz
612024-02-08T06:38:59.403Z-rw-r--r-- 1 root root 72023935 Feb 8 06:38 dsc.gz
622024-02-08T06:38:59.403Z+ echo 'input script dir contains:'
632024-02-08T06:38:59.403Zinput script dir contains:
642024-02-08T06:38:59.404Z+ ls -ltr /input/build/work/scripts
652024-02-08T06:38:59.407Ztotal 39
662024-02-08T06:38:59.407Z-rw-r--r-- 1 root root 3056 Feb 8 06:38 perf-downstairs-tick.d
672024-02-08T06:38:59.407Z-rw-r--r-- 1 root root 1738 Feb 8 06:38 test_ds.sh
682024-02-08T06:38:59.407Z-rw-r--r-- 1 root root 3746 Feb 8 06:38 test_live_repair.sh
692024-02-08T06:38:59.407Z-rw-r--r-- 1 root root 6416 Feb 8 06:38 test_repair.sh
702024-02-08T06:38:59.407Z-rw-r--r-- 1 root root 2969 Feb 8 06:38 test_replay.sh
712024-02-08T06:38:59.407Z-rw-r--r-- 1 root root 9465 Feb 8 06:38 test_up.sh
722024-02-08T06:38:59.407Z-rw-r--r-- 1 root root 2824 Feb 8 06:38 upstairs_info.d
732024-02-08T06:38:59.408Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-08T06:38:59.408Z+ echo ' chmod input script dir contains:'
752024-02-08T06:38:59.408Z chmod input script dir contains:
762024-02-08T06:38:59.408Z+ ls -ltr /input/build/work/scripts
772024-02-08T06:38:59.411Ztotal 39
782024-02-08T06:38:59.411Z-rwxr-xr-x 1 root root 3056 Feb 8 06:38 perf-downstairs-tick.d
792024-02-08T06:38:59.411Z-rwxr-xr-x 1 root root 1738 Feb 8 06:38 test_ds.sh
802024-02-08T06:38:59.411Z-rwxr-xr-x 1 root root 3746 Feb 8 06:38 test_live_repair.sh
812024-02-08T06:38:59.411Z-rwxr-xr-x 1 root root 6416 Feb 8 06:38 test_repair.sh
822024-02-08T06:38:59.411Z-rwxr-xr-x 1 root root 2969 Feb 8 06:38 test_replay.sh
832024-02-08T06:38:59.412Z-rwxr-xr-x 1 root root 9465 Feb 8 06:38 test_up.sh
842024-02-08T06:38:59.412Z-rwxr-xr-x 1 root root 2824 Feb 8 06:38 upstairs_info.d
852024-02-08T06:38:59.412Z+ banner unpack
862024-02-08T06:38:59.412Z
872024-02-08T06:38:59.412Z # # # # ##### ## #### # #
882024-02-08T06:38:59.412Z # # ## # # # # # # # # #
892024-02-08T06:38:59.412Z # # # # # # # # # # ####
902024-02-08T06:38:59.412Z # # # # # ##### ###### # # #
912024-02-08T06:38:59.412Z # # # ## # # # # # # #
922024-02-08T06:38:59.412Z #### # # # # # #### # #
932024-02-08T06:38:59.412Z
942024-02-08T06:38:59.412Z+ mkdir -p /var/tmp/bins
952024-02-08T06:38:59.415Z+ for t in "$input/bins/"*.gz
962024-02-08T06:38:59.416Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-08T06:38:59.416Z+ b=crucible-downstairs.gz
982024-02-08T06:38:59.416Z+ b=crucible-downstairs
992024-02-08T06:38:59.416Z+ gunzip
1002024-02-08T06:39:02.302Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-08T06:39:02.306Z+ for t in "$input/bins/"*.gz
1022024-02-08T06:39:02.306Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-08T06:39:02.306Z+ b=crucible-hammer.gz
1042024-02-08T06:39:02.306Z+ b=crucible-hammer
1052024-02-08T06:39:02.306Z+ gunzip
1062024-02-08T06:39:04.390Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-08T06:39:04.394Z+ for t in "$input/bins/"*.gz
1082024-02-08T06:39:04.394Z++ basename /input/build/work/bins/crutest.gz
1092024-02-08T06:39:04.394Z+ b=crutest.gz
1102024-02-08T06:39:04.394Z+ b=crutest
1112024-02-08T06:39:04.394Z+ gunzip
1122024-02-08T06:39:06.804Z+ chmod +x /var/tmp/bins/crutest
1132024-02-08T06:39:06.808Z+ for t in "$input/bins/"*.gz
1142024-02-08T06:39:06.808Z++ basename /input/build/work/bins/dsc.gz
1152024-02-08T06:39:06.808Z+ b=dsc.gz
1162024-02-08T06:39:06.808Z+ b=dsc
1172024-02-08T06:39:06.808Z+ gunzip
1182024-02-08T06:39:08.435Z+ chmod +x /var/tmp/bins/dsc
1192024-02-08T06:39:08.439Z+ export BINDIR=/var/tmp/bins
1202024-02-08T06:39:08.439Z+ BINDIR=/var/tmp/bins
1212024-02-08T06:39:08.439Z+ banner repair
1222024-02-08T06:39:08.439Z
1232024-02-08T06:39:08.439Z ##### ###### ##### ## # #####
1242024-02-08T06:39:08.439Z # # # # # # # # # #
1252024-02-08T06:39:08.439Z # # ##### # # # # # # #
1262024-02-08T06:39:08.440Z ##### # ##### ###### # #####
1272024-02-08T06:39:08.440Z # # # # # # # # #
1282024-02-08T06:39:08.440Z # # ###### # # # # # #
1292024-02-08T06:39:08.440Z
1302024-02-08T06:39:08.440Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-08T06:39:08.518ZTurn off color for downstairs dump
1322024-02-08T06:39:08.531ZCreating region directory at: /var/tmp/test_repair
1332024-02-08T06:39:08.535ZCreating dsc directory at: /tmp/dsc
1342024-02-08T06:39:08.745ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.14455935
1352024-02-08T06:39:08.769ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.09292227
1362024-02-08T06:39:08.806ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.03716561
1372024-02-08T06:39:08.806ZRegion set with 3 regions was created
1382024-02-08T06:39:08.819Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1137 1138 1139
1392024-02-08T06:39:08.819ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-08T06:39:08.844Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.101031083Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141}
1412024-02-08T06:39:08.844Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.101770038Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141}
1422024-02-08T06:39:08.844Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.101789167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141}
1432024-02-08T06:39:08.848Z{"msg":"Crucible a1883744-7f3a-4559-a403-e31d6fe54bc8 has session id: 4871c284-b0d3-4350-bfc4-a995cf1ac32f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.101804225Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1442024-02-08T06:39:08.849Z{"msg":"Upstairs opts: Upstairs UUID: a1883744-7f3a-4559-a403-e31d6fe54bc8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.101841123Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1452024-02-08T06:39:08.849Z{"msg":"Crucible stats registered with UUID: a1883744-7f3a-4559-a403-e31d6fe54bc8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.101868141Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1462024-02-08T06:39:08.849ZCrucible runtime is spawned
1472024-02-08T06:39:08.849Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102169558Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141}
1482024-02-08T06:39:08.849Z{"msg":"a1883744-7f3a-4559-a403-e31d6fe54bc8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102483525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1492024-02-08T06:39:08.849Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102514412Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1502024-02-08T06:39:08.849Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102530091Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1512024-02-08T06:39:08.849Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.10254486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1522024-02-08T06:39:08.849Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102559179Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1532024-02-08T06:39:08.849Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102586737Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1542024-02-08T06:39:08.849Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102619094Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1552024-02-08T06:39:08.849Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102645772Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1562024-02-08T06:39:08.850Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102663781Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1572024-02-08T06:39:08.850Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102760934Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1582024-02-08T06:39:08.850Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.102959909Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1592024-02-08T06:39:08.850Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.103005515Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1602024-02-08T06:39:08.850Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.103032623Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1612024-02-08T06:39:08.850Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.104843087Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1622024-02-08T06:39:08.850Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.104882834Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1632024-02-08T06:39:08.850Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.104910482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1642024-02-08T06:39:08.850Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.104948549Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1652024-02-08T06:39:08.850Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.104981767Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1662024-02-08T06:39:08.850Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.105020694Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1672024-02-08T06:39:08.858Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.106350864Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1682024-02-08T06:39:08.858Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.106388331Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1692024-02-08T06:39:08.858Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.106477815Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1702024-02-08T06:39:08.859Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107239868Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1712024-02-08T06:39:08.859Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107278334Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1722024-02-08T06:39:08.859Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107295913Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1732024-02-08T06:39:08.859Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107316252Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1742024-02-08T06:39:08.859Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107330221Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1752024-02-08T06:39:08.859Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107350909Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1762024-02-08T06:39:08.859Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107363628Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1772024-02-08T06:39:08.859Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107377987Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1782024-02-08T06:39:08.859Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107408485Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1792024-02-08T06:39:08.859Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107430313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1802024-02-08T06:39:08.859Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107586771Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1812024-02-08T06:39:08.860Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107618919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1822024-02-08T06:39:08.860Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107640677Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1832024-02-08T06:39:08.860Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107656176Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1842024-02-08T06:39:08.860Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107677155Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1852024-02-08T06:39:08.860Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107691394Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1862024-02-08T06:39:08.860Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107711962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1872024-02-08T06:39:08.860Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107725701Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1882024-02-08T06:39:08.860Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107745739Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1892024-02-08T06:39:08.860Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107758989Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1902024-02-08T06:39:08.860Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107781137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1912024-02-08T06:39:08.860Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1922024-02-08T06:39:08.860Z,"time":"2024-02-08T06:39:08.107816054Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1932024-02-08T06:39:08.860Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107842562Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1942024-02-08T06:39:08.860Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.10787174Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1952024-02-08T06:39:08.860Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107907427Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1962024-02-08T06:39:08.860Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107945404Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1972024-02-08T06:39:08.861Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107972462Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1982024-02-08T06:39:08.861Z{"msg":"a1883744-7f3a-4559-a403-e31d6fe54bc8 is now active with session: 4871c284-b0d3-4350-bfc4-a995cf1ac32f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.107993971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
1992024-02-08T06:39:08.861Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.108028678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141,"session_id":"4871c284-b0d3-4350-bfc4-a995cf1ac32f"}
2002024-02-08T06:39:08.861Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:08.108061026Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141}
2012024-02-08T06:39:08.861ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-08T06:39:08.861ZFill test
2032024-02-08T06:39:08.941ZRead and Verify all blocks (0..600 range:false)
2042024-02-08T06:39:09.011ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-08T06:39:09.011ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-08T06:39:09.015Z----------------------------------------------------------------
2072024-02-08T06:39:09.015Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-08T06:39:09.015ZStates: Active Active Active
2092024-02-08T06:39:09.015ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-08T06:39:09.015Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-08T06:39:09.015Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-08T06:39:09.015Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-08T06:39:09.015Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-08T06:39:09.015Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-08T06:39:09.015Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-08T06:39:09.015Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-08T06:39:09.015Z New 0 0 0 0
2182024-02-08T06:39:09.015Z Sent 0 0 0 0
2192024-02-08T06:39:09.015Z Done 6 6 6 18
2202024-02-08T06:39:09.016Z Skipped 0 0 0 0
2212024-02-08T06:39:09.016Z Error 0 0 0 0
2222024-02-08T06:39:09.016ZLast Flush: 1006 1006 1006
2232024-02-08T06:39:09.016ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-08T06:39:09.016ZUpstairs last five completed: 13 12 11 10 9
2252024-02-08T06:39:09.016ZCLIENT: Up:0 ds:6 act:3
2262024-02-08T06:39:13.014Z----------------------------------------------------------------
2272024-02-08T06:39:13.014Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-08T06:39:13.014ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-08T06:39:13.014ZUpstairs last five completed: 14 13 12 11 10
2302024-02-08T06:39:13.015ZCLIENT: Up:0 ds:0 act:3
2312024-02-08T06:39:13.015ZCLIENT: All crucible jobs finished, exiting program
2322024-02-08T06:39:13.020Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:12.270744412Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1141}
2332024-02-08T06:39:13.020Z
2342024-02-08T06:39:13.020ZBegin loop 1
2352024-02-08T06:39:13.020ZDownstairs to restart: 0
2362024-02-08T06:39:13.036Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293179278Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143}
2372024-02-08T06:39:13.036Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",Crucible runtime is spawned
2382024-02-08T06:39:13.044Z\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293477745Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143}
2392024-02-08T06:39:13.044Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293514042Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143}
2402024-02-08T06:39:13.044Z{"msg":"Crucible ef2af1d9-7a8f-4167-b0fb-e1ee515cb13f has session id: 8eb84eda-0ab4-457b-944d-db4389acb376","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.29354138Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2412024-02-08T06:39:13.044Z{"msg":"Upstairs opts: Upstairs UUID: ef2af1d9-7a8f-4167-b0fb-e1ee515cb13f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293579017Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2422024-02-08T06:39:13.044Z{"msg":"Crucible stats registered with UUID: ef2af1d9-7a8f-4167-b0fb-e1ee515cb13f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293596996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2432024-02-08T06:39:13.044Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293618125Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143}
2442024-02-08T06:39:13.044Z{"msg":"ef2af1d9-7a8f-4167-b0fb-e1ee515cb13f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293701918Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2452024-02-08T06:39:13.044Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293738285Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2462024-02-08T06:39:13.044Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293789962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2472024-02-08T06:39:13.044Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293862186Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2482024-02-08T06:39:13.045Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.293908333Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2492024-02-08T06:39:13.045Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.29394916Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2502024-02-08T06:39:13.045Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.294068941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2512024-02-08T06:39:13.045Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.294120997Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2522024-02-08T06:39:13.045Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.294161514Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2532024-02-08T06:39:13.045Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.294202071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2542024-02-08T06:39:13.045Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.29421974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2552024-02-08T06:39:13.045Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.294250017Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2562024-02-08T06:39:13.045Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.294289044Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2572024-02-08T06:39:13.045Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.294814085Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2582024-02-08T06:39:13.045Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.294851172Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2592024-02-08T06:39:13.045Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.294870471Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2602024-02-08T06:39:13.045Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.294906118Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2612024-02-08T06:39:13.045Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.295442488Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2622024-02-08T06:39:13.046Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.295584167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2632024-02-08T06:39:13.046Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.295732946Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2642024-02-08T06:39:13.046Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.295763093Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2652024-02-08T06:39:13.046Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.296104688Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2662024-02-08T06:39:13.046Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.296132836Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2672024-02-08T06:39:13.046Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.296147615Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2682024-02-08T06:39:13.046Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.296170953Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2692024-02-08T06:39:13.046Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.296185232Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2702024-02-08T06:39:13.046Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.296198801Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2712024-02-08T06:39:13.046Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.296399116Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2722024-02-08T06:39:13.046Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.29687254Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2732024-02-08T06:39:13.046Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.296927866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2742024-02-08T06:39:13.046Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.296963434Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2752024-02-08T06:39:13.051Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.296979652Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2762024-02-08T06:39:13.051Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.29701567Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2772024-02-08T06:39:13.051Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"Wait for a query_work_queue command to finish before sending IO
2782024-02-08T06:39:13.051Zname":"crucible","level":30,"time":"2024-02-08T06:39:12.297034278Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2792024-02-08T06:39:13.051Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297048357Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2802024-02-08T06:39:13.051Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297070705Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2812024-02-08T06:39:13.051Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297084804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2822024-02-08T06:39:13.052Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297105833Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2832024-02-08T06:39:13.052Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297120412Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2842024-02-08T06:39:13.052Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.29714015Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2852024-02-08T06:39:13.052Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297154039Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2862024-02-08T06:39:13.052Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297167098Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2872024-02-08T06:39:13.052Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297186897Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2882024-02-08T06:39:13.052Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297201156Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2892024-02-08T06:39:13.052Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297214015Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2902024-02-08T06:39:13.052Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297234073Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2912024-02-08T06:39:13.052Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297247672Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2922024-02-08T06:39:13.052Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297268171Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2932024-02-08T06:39:13.052Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.29728082Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2942024-02-08T06:39:13.052Z{"msg":"ef2af1d9-7a8f-4167-b0fb-e1ee515cb13f is now active with session: 8eb84eda-0ab4-457b-944d-db4389acb376","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297293769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2952024-02-08T06:39:13.052Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297312427Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143,"session_id":"8eb84eda-0ab4-457b-944d-db4389acb376"}
2962024-02-08T06:39:13.052Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:12.297324826Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143}
2972024-02-08T06:39:13.053ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2982024-02-08T06:39:13.053ZLoading write count information from file "/tmp/test_repair_verify.data"
2992024-02-08T06:39:13.053ZRun Repair workload
3002024-02-08T06:39:13.053Z01/30 Write block 254 len 36864 data: 2 2 2 2 2 2 2 2 2
3012024-02-08T06:39:13.053Z02/30 Write block 136 len 28672 data: 2 2 2 2 2 2 2
3022024-02-08T06:39:13.053Z03/30 Read block 519 len 32768
3032024-02-08T06:39:13.053Z04/30 Read block 119 len 40960
3042024-02-08T06:39:13.053Z05/30 Write block 397 len 20480 data: 2 2 2 2 2
3052024-02-08T06:39:13.053Z06/30 Read block 490 len 36864
3062024-02-08T06:39:13.053Z07/30 Read block 548 len 4096
3072024-02-08T06:39:13.053Z08/30 Write block 149 len 28672 data: 2 2 2 2 2 2 2
3082024-02-08T06:39:13.057Z09/30 Write block 164 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3092024-02-08T06:39:13.057Z10/30 Flush
3102024-02-08T06:39:13.089Z11/30 Write block 24 len 16384 data: 2 2 2 2
3112024-02-08T06:39:13.092Z12/30 Read block 384 len 28672
3122024-02-08T06:39:13.092Z13/30 Read block 414 len 36864
3132024-02-08T06:39:13.092Z14/30 Write block 376 len 36864 data: 2 2 2 2 2 2 2 2 2
3142024-02-08T06:39:13.092Z15/30 Read block 219 len 20480
3152024-02-08T06:39:13.095Z16/30 Write block 287 len 24576 data: 2 2 2 2 2 2
3162024-02-08T06:39:13.095Z17/30 Write block 553 len 28672 data: 2 2 2 2 2 2 2
3172024-02-08T06:39:13.096Z18/30 Read block 68 len 20480
3182024-02-08T06:39:13.099Z19/30 Read block 489 len 24576
3192024-02-08T06:39:13.099Z20/30 Write block 345 len 28672 data: 2 2 2 2 2 2 2
3202024-02-08T06:39:13.099Z21/30 Read block 23 len 32768
3212024-02-08T06:39:13.099Z22/30 Read block 467 len 12288
3222024-02-08T06:39:13.102Z23/30 Read block 452 len 12288
3232024-02-08T06:39:13.102Z24/30 Read block 581 len 20480
3242024-02-08T06:39:13.102Z25/30 Read block 305 len 32768
3252024-02-08T06:39:13.102Z26/30 Write block 211 len 16384 data: 2 2 2 2
3262024-02-08T06:39:13.105Z27/30 Write block 359 len 4096 data: 2
3272024-02-08T06:39:13.106Z28/30 Write block 262 len 32768 data: 3 2 2 2 2 2 2 2
3282024-02-08T06:39:13.106Z29/30 Read block 296 len 16384
3292024-02-08T06:39:13.106Z30/30 Read block 19 len 36864
3302024-02-08T06:39:13.109Z----------------------------------------------------------------
3312024-02-08T06:39:13.110Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3322024-02-08T06:39:13.110ZStates: Active Active Active
3332024-02-08T06:39:13.110ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3342024-02-08T06:39:13.110Z 1 Acked 1000 Write 9 Done Done Done false
3352024-02-08T06:39:13.110Z 2 Acked 1001 Write 7 Sent Done Done false
3362024-02-08T06:39:13.110Z 3 Acked 1002 Read 8 Sent Done Done false
3372024-02-08T06:39:13.110Z 4 Acked 1003 Read 10 Sent Done Done false
3382024-02-08T06:39:13.110Z 5 Acked 1004 Write 5 Sent Done Done false
3392024-02-08T06:39:13.110Z 6 Acked 1005 Read 9 Sent Done Done false
3402024-02-08T06:39:13.110Z 7 Acked 1006 Read 1 Sent Done Done false
3412024-02-08T06:39:13.110Z 8 Acked 1007 Write 7 Sent Done Done false
3422024-02-08T06:39:13.110Z 9 Acked 1008 Write 10 Sent Done Done false
3432024-02-08T06:39:13.110Z 10 Acked 1009 Flush 0 Sent Done Done false
3442024-02-08T06:39:13.110Z 11 Acked 1010 Write 4 Sent Done Done false
3452024-02-08T06:39:13.110Z 12 Acked 1011 Read 7 Sent Done Done false
3462024-02-08T06:39:13.111Z 13 Acked 1012 Read 9 Sent Done Done false
3472024-02-08T06:39:13.111Z 14 Acked 1013 Write 9 Sent Done Done false
3482024-02-08T06:39:13.111Z 15 Acked 1014 Read 5 Sent Done Done false
3492024-02-08T06:39:13.111Z 16 Acked 1015 Write 6 Sent Done Done false
3502024-02-08T06:39:13.111Z 17 Acked 1016 Write 7 Sent Done Done false
3512024-02-08T06:39:13.111Z 18 Acked 1017 Read 5 Sent Done Done false
3522024-02-08T06:39:13.111Z 19 Acked 1018 Read 6 Sent Done Done false
3532024-02-08T06:39:13.111Z 20 Acked 1019 Write 7 Sent Done Done false
3542024-02-08T06:39:13.111Z 21 Acked 1020 Read 8 Sent Done Done false
3552024-02-08T06:39:13.111Z 22 Acked 1021 Read 3 Sent Done Done false
3562024-02-08T06:39:13.111Z 23 Acked 1022 Read 3 Sent Done Done false
3572024-02-08T06:39:13.111Z 24 Acked 1023 Read 5 Sent Done Done false
3582024-02-08T06:39:13.111Z 25 Acked 1024 Read 8 Sent Done Done false
3592024-02-08T06:39:13.111Z 26 Acked 1025 Write 4 Sent Done Done false
3602024-02-08T06:39:13.111Z 27 Acked 1026 Write 1 Sent Done Done false
3612024-02-08T06:39:13.112Z 28 Acked 1027 Write 8 Sent Done Done false
3622024-02-08T06:39:13.112Z 29 Acked 1028 Read 4 Sent Done Done false
3632024-02-08T06:39:13.112Z 30 Acked 1029 Read 9 Sent Done Done false
3642024-02-08T06:39:13.112Z STATES DS:0 DS:1 DS:2 TOTAL
3652024-02-08T06:39:13.112Z New 0 0 0 0
3662024-02-08T06:39:13.112Z Sent 29 0 0 29
3672024-02-08T06:39:13.112Z Done 1 30 30 61
3682024-02-08T06:39:13.112Z Skipped 0 0 0 0
3692024-02-08T06:39:13.112Z Error 0 0 0 0
3702024-02-08T06:39:13.112ZLast Flush: 0 1009 1009
3712024-02-08T06:39:13.112ZDownstairs last five completed:
3722024-02-08T06:39:13.112ZUpstairs last five completed: 30 29 28 27 26
3732024-02-08T06:39:13.112Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:12.363931856Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1143}
3742024-02-08T06:39:13.112ZWrote out file "/tmp/test_repair_verify.data"
3752024-02-08T06:39:13.116Z
3762024-02-08T06:39:13.116ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3772024-02-08T06:39:13.130ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3782024-02-08T06:39:13.130Z 0 000-019 1 1 1 1 1 1 F F F
3792024-02-08T06:39:13.134Z 1 020-039 1 1 1 1 1 1 F T T <---
3802024-02-08T06:39:13.134Z 2 040-059 1 1 1 1 1 1 F F F
3812024-02-08T06:39:13.134Z 3 060-079 1 1 1 1 1 1 F F F
3822024-02-08T06:39:13.134Z 4 080-099 1 1 1 1 1 1 F F F
3832024-02-08T06:39:13.135Z 5 100-119 1 1 1 1 1 1 F F F
3842024-02-08T06:39:13.135Z 6 120-139 1 2 2 1 2 2 F F F <---
3852024-02-08T06:39:13.135Z 7 140-159 1 2 2 1 2 2 F F F <---
3862024-02-08T06:39:13.135Z 8 160-179 1 2 2 1 2 2 F F F <---
3872024-02-08T06:39:13.135Z 9 180-199 1 1 1 1 1 1 F F F
3882024-02-08T06:39:13.135Z 10 200-219 1 1 1 1 1 1 F T T <---
3892024-02-08T06:39:13.135Z 11 220-239 1 1 1 1 1 1 F F F
3902024-02-08T06:39:13.135Z 12 240-259 1 2 2 1 2 2 T F F <---
3912024-02-08T06:39:13.135Z 13 260-279 1 2 2 1 2 2 T T T <---
3922024-02-08T06:39:13.135Z 14 280-299 1 1 1 1 1 1 F T T <---
3932024-02-08T06:39:13.135Z 15 300-319 1 1 1 1 1 1 F F F
3942024-02-08T06:39:13.135Z 16 320-339 1 1 1 1 1 1 F F F
3952024-02-08T06:39:13.135Z 17 340-359 1 1 1 1 1 1 F T T <---
3962024-02-08T06:39:13.135Z 18 360-379 1 1 1 1 1 1 F T T <---
3972024-02-08T06:39:13.135Z 19 380-399 1 2 2 1 2 2 F T T <---
3982024-02-08T06:39:13.135Z 20 400-419 1 2 2 1 2 2 F F F <---
3992024-02-08T06:39:13.136Z 21 420-439 1 1 1 1 1 1 F F F
4002024-02-08T06:39:13.136Z 22 440-459 1 1 1 1 1 1 F F F
4012024-02-08T06:39:13.136Z 23 460-479 1 1 1 1 1 1 F F F
4022024-02-08T06:39:13.136Z 24 480-499 1 1 1 1 1 1 F F F
4032024-02-08T06:39:13.136Z 25 500-519 1 1 1 1 1 1 F F F
4042024-02-08T06:39:13.136Z 26 520-539 1 1 1 1 1 1 F F F
4052024-02-08T06:39:13.136Z 27 540-559 1 1 1 1 1 1 F T T <---
4062024-02-08T06:39:13.136Z 28 560-579 1 1 1 1 1 1 F F F
4072024-02-08T06:39:13.136Z 29 580-599 1 1 1 1 1 1 F F F
4082024-02-08T06:39:13.136ZMax gen: 2, Max flush: 2
4092024-02-08T06:39:13.136ZError: Difference in extent metadata found!
4102024-02-08T06:39:13.136ZA Difference in extent metadata is expected here
4112024-02-08T06:39:13.136ZOn loop 1
4122024-02-08T06:39:13.136Z
4132024-02-08T06:39:13.140ZVerifying data now
4142024-02-08T06:39:19.567ZNo change after verify
4152024-02-08T06:39:19.567ZLoop: 1 Downstairs dump after verify (and repair):
4162024-02-08T06:39:19.582ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4172024-02-08T06:39:19.582Z 0 000-019 1 1 1 1 1 1 F F F
4182024-02-08T06:39:19.582Z 1 020-039 3 3 3 3 3 3 F F F
4192024-02-08T06:39:19.582Z 2 040-059 1 1 1 1 1 1 F F F
4202024-02-08T06:39:19.582Z 3 060-079 1 1 1 1 1 1 F F F
4212024-02-08T06:39:19.582Z 4 080-099 1 1 1 1 1 1 F F F
4222024-02-08T06:39:19.582Z 5 100-119 1 1 1 1 1 1 F F F
4232024-02-08T06:39:19.582Z 6 120-139 2 2 2 2 2 2 F F F
4242024-02-08T06:39:19.582Z 7 140-159 2 2 2 2 2 2 F F F
4252024-02-08T06:39:19.582Z 8 160-179 2 2 2 2 2 2 F F F
4262024-02-08T06:39:19.582Z 9 180-199 1 1 1 1 1 1 F F F
4272024-02-08T06:39:19.582Z 10 200-219 3 3 3 3 3 3 F F F
4282024-02-08T06:39:19.582Z 11 220-239 1 1 1 1 1 1 F F F
4292024-02-08T06:39:19.583Z 12 240-259 2 2 2 2 2 2 F F F
4302024-02-08T06:39:19.583Z 13 260-279 3 3 3 3 3 3 F F F
4312024-02-08T06:39:19.586Z 14 280-299 3 3 3 3 3 3 F F F
4322024-02-08T06:39:19.586Z 15 300-319 1 1 1 1 1 1 F F F
4332024-02-08T06:39:19.586Z 16 320-339 1 1 1 1 1 1 F F F
4342024-02-08T06:39:19.586Z 17 340-359 3 3 3 3 3 3 F F F
4352024-02-08T06:39:19.586Z 18 360-379 3 3 3 3 3 3 F F F
4362024-02-08T06:39:19.586Z 19 380-399 3 3 3 3 3 3 F F F
4372024-02-08T06:39:19.586Z 20 400-419 2 2 2 2 2 2 F F F
4382024-02-08T06:39:19.586Z 21 420-439 1 1 1 1 1 1 F F F
4392024-02-08T06:39:19.587Z 22 440-459 1 1 1 1 1 1 F F F
4402024-02-08T06:39:19.587Z 23 460-479 1 1 1 1 1 1 F F F
4412024-02-08T06:39:19.587Z 24 480-499 1 1 1 1 1 1 F F F
4422024-02-08T06:39:19.587Z 25 500-519 1 1 1 1 1 1 F F F
4432024-02-08T06:39:19.587Z 26 520-539 1 1 1 1 1 1 F F F
4442024-02-08T06:39:19.587Z 27 540-559 3 3 3 3 3 3 F F F
4452024-02-08T06:39:19.587Z 28 560-579 1 1 1 1 1 1 F F F
4462024-02-08T06:39:19.587Z 29 580-599 1 1 1 1 1 1 F F F
4472024-02-08T06:39:19.587ZMax gen: 3, Max flush: 3
4482024-02-08T06:39:19.587Z
4492024-02-08T06:39:19.587ZBegin loop 2
4502024-02-08T06:39:19.587ZDownstairs to restart: 2
4512024-02-08T06:39:19.601Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858160492Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151}
4522024-02-08T06:39:19.601Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858430981Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151}
4532024-02-08T06:39:19.602ZCrucible runtime is spawned
4542024-02-08T06:39:19.607Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858479838Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151}
4552024-02-08T06:39:19.607Z{"msg":"Crucible a81835f5-512d-4c08-a15c-e685c8e3022d has session id: fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858521935Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4562024-02-08T06:39:19.607Z{"msg":"Upstairs opts: Upstairs UUID: a81835f5-512d-4c08-a15c-e685c8e3022d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858563162Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4572024-02-08T06:39:19.607Z{"msg":"Crucible stats registered with UUID: a81835f5-512d-4c08-a15c-e685c8e3022d","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858589949Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4582024-02-08T06:39:19.608Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858606418Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151}
4592024-02-08T06:39:19.608Z{"msg":"a81835f5-512d-4c08-a15c-e685c8e3022d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858667484Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4602024-02-08T06:39:19.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858702521Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4612024-02-08T06:39:19.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858733299Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4622024-02-08T06:39:19.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858771236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4632024-02-08T06:39:19.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858792784Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4642024-02-08T06:39:19.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858824302Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4652024-02-08T06:39:19.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.85885007Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4662024-02-08T06:39:19.608Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858880778Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4672024-02-08T06:39:19.608Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858921815Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4682024-02-08T06:39:19.608Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858949362Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4692024-02-08T06:39:19.608Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.858975211Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4702024-02-08T06:39:19.608Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.859022997Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4712024-02-08T06:39:19.608Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.859070964Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4722024-02-08T06:39:19.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.859608343Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4732024-02-08T06:39:19.609Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.85964655Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4742024-02-08T06:39:19.609Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.859671828Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4752024-02-08T06:39:19.609Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.859694906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4762024-02-08T06:39:19.609Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.860302811Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4772024-02-08T06:39:19.609Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.860362736Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4782024-02-08T06:39:19.609Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.860800014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4792024-02-08T06:39:19.609Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.860839981Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4802024-02-08T06:39:19.609Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.860877838Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4812024-02-08T06:39:19.609Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.860946423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4822024-02-08T06:39:19.609Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.860969231Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4832024-02-08T06:39:19.609Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.86098349Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4842024-02-08T06:39:19.609Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.860997309Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4852024-02-08T06:39:19.609Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.861032796Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4862024-02-08T06:39:19.609Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.861521769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4872024-02-08T06:39:19.609Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.861964046Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4882024-02-08T06:39:19.609Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862001443Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4892024-02-08T06:39:19.610Z{"msg":"[0]R flush_numbers[0..12]: [1, 3, 1, 1, 1, 1, 2, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862018042Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4902024-02-08T06:39:19.610Z{"msg":"[0]R generation[0..12]: [1, 3, 1, 1, 1, 1, 2, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862033751Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4912024-02-08T06:39:19.610Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.86205138Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4922024-02-08T06:39:19.610Z{"msg":"[1]R flush_numbers[0..12]: [1, 3, 1, 1, 1, 1, 2, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862073908Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4932024-02-08T06:39:19.610Z{"msg":"[1]R generation[0..12]: [1, 3, 1, 1, 1, 1, 2, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862088227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4942024-02-08T06:39:19.614Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862102346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4952024-02-08T06:39:19.614Z{"msg":"[2]R flush_numbers[0..12]: [1, 3, 1, 1, 1, 1, 2, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862123584Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4962024-02-08T06:39:19.614Z{"msg":"[2]R generation[0..12]: [1, 3, 1, 1, 1, 1, 2, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862138053Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4972024-02-08T06:39:19.615Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862153402Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4982024-02-08T06:39:19.615Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862167631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
4992024-02-08T06:39:19.615Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.86218127Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5002024-02-08T06:39:19.615ZWait for a query_work_queue command to finish before sending IO
5012024-02-08T06:39:19.615Z{"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862204578Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5022024-02-08T06:39:19.615Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862219707Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5032024-02-08T06:39:19.615Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862234296Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5042024-02-08T06:39:19.615Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862254105Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5052024-02-08T06:39:19.615Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862268023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5062024-02-08T06:39:19.615Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862289102Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5072024-02-08T06:39:19.615Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862304011Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5082024-02-08T06:39:19.615Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.86231651Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5092024-02-08T06:39:19.615Z{"msg":"a81835f5-512d-4c08-a15c-e685c8e3022d is now active with session: fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862329519Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5102024-02-08T06:39:19.615Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862342228Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151,"session_id":"fbb82ed5-03a3-4a4b-998b-f72cfa76f6f2"}
5112024-02-08T06:39:19.615Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:18.862354547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151}
5122024-02-08T06:39:19.616ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5132024-02-08T06:39:19.616ZLoading write count information from file "/tmp/test_repair_verify.data"
5142024-02-08T06:39:19.616ZRun Repair workload
5152024-02-08T06:39:19.616Z01/30 Write block 299 len 16384 data: 2 2 2 2
5162024-02-08T06:39:19.616Z02/30 Read block 150 len 20480
5172024-02-08T06:39:19.616Z03/30 Read block 151 len 32768
5182024-02-08T06:39:19.616Z04/30 Write block 422 len 12288 data: 2 2 2
5192024-02-08T06:39:19.616Z05/30 Read block 209 len 40960
5202024-02-08T06:39:19.616Z06/30 Write block 455 len 16384 data: 2 2 2 2
5212024-02-08T06:39:19.616Z07/30 Write block 225 len 20480 data: 2 2 2 2 2
5222024-02-08T06:39:19.616Z08/30 Write block 299 len 8192 data: 3 3
5232024-02-08T06:39:19.616Z09/30 Flush
5242024-02-08T06:39:19.620Z10/30 Write block 395 len 8192 data: 2 2
5252024-02-08T06:39:19.620Z11/30 Write block 287 len 4096 data: 3
5262024-02-08T06:39:19.620Z12/30 Read block 309 len 8192
5272024-02-08T06:39:19.620Z13/30 Write block 480 len 16384 data: 2 2 2 2
5282024-02-08T06:39:19.620Z14/30 Write block 333 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5292024-02-08T06:39:19.620Z15/30 Flush
5302024-02-08T06:39:19.626Z16/30 Write block 149 len 36864 data: 3 3 3 3 3 3 3 2 2
5312024-02-08T06:39:19.626Z17/30 Read block 28 len 24576
5322024-02-08T06:39:19.629Z18/30 Read block 190 len 12288
5332024-02-08T06:39:19.629Z19/30 Read block 112 len 24576
5342024-02-08T06:39:19.629Z20/30 Read block 36 len 4096
5352024-02-08T06:39:19.629Z21/30 Write block 92 len 24576 data: 2 2 2 2 2 2
5362024-02-08T06:39:19.629Z22/30 Read block 577 len 24576
5372024-02-08T06:39:19.630Z23/30 Flush
5382024-02-08T06:39:19.633Z24/30 Write block 384 len 24576 data: 3 2 2 2 2 2
5392024-02-08T06:39:19.633Z25/30 Flush
5402024-02-08T06:39:19.633Z26/30 Write block 193 len 4096 data: 2
5412024-02-08T06:39:19.637Z27/30 Write block 576 len 4096 data: 2
5422024-02-08T06:39:19.637Z28/30 Read block 243 len 32768
5432024-02-08T06:39:19.637Z29/30 Write block 162 len 16384 data: 2 2 3 3
5442024-02-08T06:39:19.637Z30/30 Write block 265 len 28672 data: 3 3 3 3 3 2 2
5452024-02-08T06:39:19.637Z----------------------------------------------------------------
5462024-02-08T06:39:19.637Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
5472024-02-08T06:39:19.637ZStates: Active Active Active
5482024-02-08T06:39:19.637ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5492024-02-08T06:39:19.637Z 1 Acked 1000 Write 4 Done Done Sent false
5502024-02-08T06:39:19.637Z 2 Acked 1001 Read 5 Done Done Sent false
5512024-02-08T06:39:19.637Z 3 Acked 1002 Read 8 Done Done Sent false
5522024-02-08T06:39:19.638Z 4 Acked 1003 Write 3 Done Done Sent false
5532024-02-08T06:39:19.638Z 5 Acked 1004 Read 10 Done Done Sent false
5542024-02-08T06:39:19.638Z 6 Acked 1005 Write 4 Done Done Sent false
5552024-02-08T06:39:19.638Z 7 Acked 1006 Write 5 Done Done Sent false
5562024-02-08T06:39:19.638Z 8 Acked 1007 Write 2 Done Done Sent false
5572024-02-08T06:39:19.638Z 9 Acked 1008 Flush 0 Done Done Sent false
5582024-02-08T06:39:19.638Z 10 Acked 1009 Write 2 Done Done Sent false
5592024-02-08T06:39:19.638Z 11 Acked 1010 Write 1 Done Done Sent false
5602024-02-08T06:39:19.638Z 12 Acked 1011 Read 2 Done Done Sent false
5612024-02-08T06:39:19.638Z 13 Acked 1012 Write 4 Done Done Sent false
5622024-02-08T06:39:19.638Z 14 Acked 1013 Write 10 Done Done Sent false
5632024-02-08T06:39:19.638Z 15 Acked 1014 Flush 0 Done Done Sent false
5642024-02-08T06:39:19.638Z 16 Acked 1015 Write 9 Done Done Sent false
5652024-02-08T06:39:19.638Z 17 Acked 1016 Read 6 Done Done Sent false
5662024-02-08T06:39:19.638Z 18 Acked 1017 Read 3 Done Done Sent false
5672024-02-08T06:39:19.638Z 19 Acked 1018 Read 6 Done Done Sent false
5682024-02-08T06:39:19.639Z 20 Acked 1019 Read 1 Done Done Sent false
5692024-02-08T06:39:19.639Z 21 Acked 1020 Write 6 Done Done Sent false
5702024-02-08T06:39:19.639Z 22 Acked 1021 Read 6 Done Done Sent false
5712024-02-08T06:39:19.639Z 23 Acked 1022 Flush 0 Done Done Sent false
5722024-02-08T06:39:19.639Z 24 Acked 1023 Write 6 Done Done Sent false
5732024-02-08T06:39:19.639Z 25 Acked 1024 Flush 0 Done Done Sent false
5742024-02-08T06:39:19.639Z 26 Acked 1025 Write 1 Done Done Sent false
5752024-02-08T06:39:19.639Z 27 Acked 1026 Write 1 Done Done Sent false
5762024-02-08T06:39:19.639Z 28 Acked 1027 Read 8 Done Done Sent false
5772024-02-08T06:39:19.639Z 29 Acked 1028 Write 4 Done Done Sent false
5782024-02-08T06:39:19.639Z 30 Acked 1029 Write 7 Sent Sent Sent false
5792024-02-08T06:39:19.639Z STATES DS:0 DS:1 DS:2 TOTAL
5802024-02-08T06:39:19.639Z New 0 0 0 0
5812024-02-08T06:39:19.639Z Sent 1 1 30 32
5822024-02-08T06:39:19.639Z Done 29 29 0 58
5832024-02-08T06:39:19.640Z Skipped 0 0 0 0
5842024-02-08T06:39:19.640Z Error 0 0 0 0
5852024-02-08T06:39:19.640ZLast Flush: 1024 1024 0
5862024-02-08T06:39:19.640ZDownstairs last five completed:
5872024-02-08T06:39:19.640ZUpstairs last five completed: 30 29 28 27 26
5882024-02-08T06:39:19.640Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:18.893052752Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1151}
5892024-02-08T06:39:19.640ZWrote out file "/tmp/test_repair_verify.data"
5902024-02-08T06:39:19.645Z
5912024-02-08T06:39:19.645ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5922024-02-08T06:39:19.658ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5932024-02-08T06:39:19.658Z 0 000-019 1 1 1 1 1 1 F F F
5942024-02-08T06:39:19.658Z 1 020-039 3 3 3 3 3 3 F F F
5952024-02-08T06:39:19.658Z 2 040-059 1 1 1 1 1 1 F F F
5962024-02-08T06:39:19.659Z 3 060-079 1 1 1 1 1 1 F F F
5972024-02-08T06:39:19.659Z 4 080-099 4 4 1 6 6 1 F F F <---
5982024-02-08T06:39:19.659Z 5 100-119 1 1 1 1 1 1 F F F
5992024-02-08T06:39:19.659Z 6 120-139 2 2 2 2 2 2 F F F
6002024-02-08T06:39:19.662Z 7 140-159 4 4 2 6 6 2 F F F <---
6012024-02-08T06:39:19.662Z 8 160-179 2 2 2 2 2 2 T T F <---
6022024-02-08T06:39:19.663Z 9 180-199 1 1 1 1 1 1 T T F <---
6032024-02-08T06:39:19.663Z 10 200-219 3 3 3 3 3 3 F F F
6042024-02-08T06:39:19.663Z 11 220-239 4 4 1 4 4 1 F F F <---
6052024-02-08T06:39:19.663Z 12 240-259 2 2 2 2 2 2 F F F
6062024-02-08T06:39:19.663Z 13 260-279 3 3 3 3 3 3 T T F <---
6072024-02-08T06:39:19.663Z 14 280-299 4 4 3 5 5 3 F F F <---
6082024-02-08T06:39:19.663Z 15 300-319 4 4 1 4 4 1 F F F <---
6092024-02-08T06:39:19.663Z 16 320-339 4 4 1 5 5 1 F F F <---
6102024-02-08T06:39:19.663Z 17 340-359 4 4 3 5 5 3 F F F <---
6112024-02-08T06:39:19.663Z 18 360-379 3 3 3 3 3 3 F F F
6122024-02-08T06:39:19.663Z 19 380-399 4 4 3 7 7 3 F F F <---
6132024-02-08T06:39:19.663Z 20 400-419 2 2 2 2 2 2 F F F
6142024-02-08T06:39:19.663Z 21 420-439 4 4 1 4 4 1 F F F <---
6152024-02-08T06:39:19.663Z 22 440-459 4 4 1 4 4 1 F F F <---
6162024-02-08T06:39:19.663Z 23 460-479 1 1 1 1 1 1 F F F
6172024-02-08T06:39:19.664Z 24 480-499 4 4 1 5 5 1 F F F <---
6182024-02-08T06:39:19.664Z 25 500-519 1 1 1 1 1 1 F F F
6192024-02-08T06:39:19.664Z 26 520-539 1 1 1 1 1 1 F F F
6202024-02-08T06:39:19.664Z 27 540-559 3 3 3 3 3 3 F F F
6212024-02-08T06:39:19.664Z 28 560-579 1 1 1 1 1 1 T T F <---
6222024-02-08T06:39:19.664Z 29 580-599 1 1 1 1 1 1 F F F
6232024-02-08T06:39:19.664ZMax gen: 4, Max flush: 7
6242024-02-08T06:39:19.664ZError: Difference in extent metadata found!
6252024-02-08T06:39:19.664ZA Difference in extent metadata is expected here
6262024-02-08T06:39:19.664ZOn loop 2
6272024-02-08T06:39:19.664Z
6282024-02-08T06:39:19.667ZVerifying data now
6292024-02-08T06:39:24.393ZNo change after verify
6302024-02-08T06:39:24.394ZLoop: 2 Downstairs dump after verify (and repair):
6312024-02-08T06:39:24.408ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6322024-02-08T06:39:24.409Z 0 000-019 1 1 1 1 1 1 F F F
6332024-02-08T06:39:24.409Z 1 020-039 3 3 3 3 3 3 F F F
6342024-02-08T06:39:24.409Z 2 040-059 1 1 1 1 1 1 F F F
6352024-02-08T06:39:24.409Z 3 060-079 1 1 1 1 1 1 F F F
6362024-02-08T06:39:24.410Z 4 080-099 4 4 4 6 6 6 F F F
6372024-02-08T06:39:24.410Z 5 100-119 1 1 1 1 1 1 F F F
6382024-02-08T06:39:24.410Z 6 120-139 2 2 2 2 2 2 F F F
6392024-02-08T06:39:24.410Z 7 140-159 4 4 4 6 6 6 F F F
6402024-02-08T06:39:24.410Z 8 160-179 5 5 5 8 8 8 F F F
6412024-02-08T06:39:24.411Z 9 180-199 5 5 5 8 8 8 F F F
6422024-02-08T06:39:24.411Z 10 200-219 3 3 3 3 3 3 F F F
6432024-02-08T06:39:24.425Z 11 220-239 4 4 4 4 4 4 F F F
6442024-02-08T06:39:24.426Z 12 240-259 2 2 2 2 2 2 F F F
6452024-02-08T06:39:24.426Z 13 260-279 5 5 5 8 8 8 F F F
6462024-02-08T06:39:24.426Z 14 280-299 4 4 4 5 5 5 F F F
6472024-02-08T06:39:24.427Z 15 300-319 4 4 4 4 4 4 F F F
6482024-02-08T06:39:24.427Z 16 320-339 4 4 4 5 5 5 F F F
6492024-02-08T06:39:24.427Z 17 340-359 4 4 4 5 5 5 F F F
6502024-02-08T06:39:24.427Z 18 360-379 3 3 3 3 3 3 F F F
6512024-02-08T06:39:24.427Z 19 380-399 4 4 4 7 7 7 F F F
6522024-02-08T06:39:24.427Z 20 400-419 2 2 2 2 2 2 F F F
6532024-02-08T06:39:24.428Z 21 420-439 4 4 4 4 4 4 F F F
6542024-02-08T06:39:24.428Z 22 440-459 4 4 4 4 4 4 F F F
6552024-02-08T06:39:24.428Z 23 460-479 1 1 1 1 1 1 F F F
6562024-02-08T06:39:24.428Z 24 480-499 4 4 4 5 5 5 F F F
6572024-02-08T06:39:24.428Z 25 500-519 1 1 1 1 1 1 F F F
6582024-02-08T06:39:24.428Z 26 520-539 1 1 1 1 1 1 F F F
6592024-02-08T06:39:24.429Z 27 540-559 3 3 3 3 3 3 F F F
6602024-02-08T06:39:24.429Z 28 560-579 5 5 5 8 8 8 F F F
6612024-02-08T06:39:24.429Z 29 580-599 1 1 1 1 1 1 F F F
6622024-02-08T06:39:24.429ZMax gen: 5, Max flush: 8
6632024-02-08T06:39:24.429Z
6642024-02-08T06:39:24.429ZBegin loop 3
6652024-02-08T06:39:24.430ZDownstairs to restart: 1
6662024-02-08T06:39:24.442Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.684249189Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159}
6672024-02-08T06:39:24.442Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.684553606Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159}
6682024-02-08T06:39:24.443Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.684595873Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159}
6692024-02-08T06:39:24.443Z{"msg":"Crucible 1a292212-1bc8-4b81-9ebe-cc03afb6498c has session id: 2d0e5fef-bd39-4458-bd07-5543945d26e9","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.684622161Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6702024-02-08T06:39:24.443Z{"msg":"Upstairs opts: Upstairs UUID: 1a292212-1bc8-4b81-9ebe-cc03afb6498c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
6712024-02-08T06:39:24.443Z,"time":"2024-02-08T06:39:23.684657738Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6722024-02-08T06:39:24.444Z{"msg":"Crucible stats registered with UUID: 1a292212-1bc8-4b81-9ebe-cc03afb6498c","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.684713784Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6732024-02-08T06:39:24.444Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.684755911Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159}
6742024-02-08T06:39:24.444Z{"msg":"1a292212-1bc8-4b81-9ebe-cc03afb6498c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.684858243Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6752024-02-08T06:39:24.444Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.684898591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6762024-02-08T06:39:24.445Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.684957466Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6772024-02-08T06:39:24.445Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.68504007Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6782024-02-08T06:39:24.445Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.68516662Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6792024-02-08T06:39:24.445Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.685214177Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6802024-02-08T06:39:24.445Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.685240895Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6812024-02-08T06:39:24.446Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.68530591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6822024-02-08T06:39:24.446Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.685331538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6832024-02-08T06:39:24.446Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.685354956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6842024-02-08T06:39:24.446Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.685377875Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6852024-02-08T06:39:24.446Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.685418781Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6862024-02-08T06:39:24.447Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.685455539Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6872024-02-08T06:39:24.447Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.685959261Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6882024-02-08T06:39:24.447Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.686002988Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6892024-02-08T06:39:24.447Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.686030246Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6902024-02-08T06:39:24.447Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.686058063Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6912024-02-08T06:39:24.448Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.686598683Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6922024-02-08T06:39:24.448Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.686825186Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6932024-02-08T06:39:24.448Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.686872872Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6942024-02-08T06:39:24.448Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.686912559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6952024-02-08T06:39:24.449Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.687093686Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6962024-02-08T06:39:24.449Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.687139862Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6972024-02-08T06:39:24.449Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.68716736Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6982024-02-08T06:39:24.449Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.687396383Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
6992024-02-08T06:39:24.449Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.68743157Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7002024-02-08T06:39:24.450Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.687447489Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7012024-02-08T06:39:24.450Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.687461708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7022024-02-08T06:39:24.450Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.687905305Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7032024-02-08T06:39:24.450Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.687944922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7042024-02-08T06:39:24.450Z{"msg":"[0]R flush_numbers[0..12]: [1, 3, 1, 1, 6, 1, 2, 6, 8, 8, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.687960641Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7052024-02-08T06:39:24.451Z{"msg":"[0]R generation[0..12]: [1, 3, 1, 1, 4, 1, 2, 4, 5, 5, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.6879755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7062024-02-08T06:39:24.451Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.687990708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7072024-02-08T06:39:24.451Z{"msg":"[1]R flush_numbers[0..12]: [1, 3, 1, 1, 6, 1, 2, 6, 8, 8, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688004897Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7082024-02-08T06:39:24.451Z{"msg":"[1]R generation[0..12]: [1, 3, 1, 1, 4, 1, 2, 4, 5, 5, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688026716Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7092024-02-08T06:39:24.451Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688041524Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7102024-02-08T06:39:24.452Z{"msg":"[2]R flush_numbers[0..12]: [1, 3, 1, 1, 6, 1, 2, 6, 8, 8, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688056413Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7112024-02-08T06:39:24.452Z{"msg":"[2]R generation[0..12]: [1, 3, 1, 1, 4, 1, 2, 4, 5, 5, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688070132Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7122024-02-08T06:39:24.452Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688084371Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7132024-02-08T06:39:24.452Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688109939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7142024-02-08T06:39:24.452Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688131038Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO
7152024-02-08T06:39:24.453Z","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7162024-02-08T06:39:24.453Z{"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688148986Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7172024-02-08T06:39:24.453Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688162165Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7182024-02-08T06:39:24.453Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688183444Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7192024-02-08T06:39:24.453Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688195913Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7202024-02-08T06:39:24.454Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688209732Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7212024-02-08T06:39:24.454Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.68822994Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7222024-02-08T06:39:24.454Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688243539Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7232024-02-08T06:39:24.454Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688255798Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7242024-02-08T06:39:24.454Z{"msg":"1a292212-1bc8-4b81-9ebe-cc03afb6498c is now active with session: 2d0e5fef-bd39-4458-bd07-5543945d26e9","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688275207Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7252024-02-08T06:39:24.455Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688287446Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159,"session_id":"2d0e5fef-bd39-4458-bd07-5543945d26e9"}
7262024-02-08T06:39:24.455Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:23.688299785Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159}
7272024-02-08T06:39:24.455ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7282024-02-08T06:39:24.455ZLoading write count information from file "/tmp/test_repair_verify.data"
7292024-02-08T06:39:24.456ZRun Repair workload
7302024-02-08T06:39:24.456Z01/30 Flush
7312024-02-08T06:39:24.456Z02/30 Write block 37 len 32768 data: 2 2 2 2 2 2 2 2
7322024-02-08T06:39:24.466Z03/30 Read block 111 len 24576
7332024-02-08T06:39:24.466Z04/30 Read block 34 len 32768
7342024-02-08T06:39:24.466Z05/30 Write block 560 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7352024-02-08T06:39:24.467Z06/30 Write block 46 len 12288 data: 2 2 2
7362024-02-08T06:39:24.467Z07/30 Read block 487 len 32768
7372024-02-08T06:39:24.467Z08/30 Read block 457 len 12288
7382024-02-08T06:39:24.468Z09/30 Read block 92 len 40960
7392024-02-08T06:39:24.468Z10/30 Write block 472 len 16384 data: 2 2 2 2
7402024-02-08T06:39:24.468Z11/30 Flush
7412024-02-08T06:39:24.469Z12/30 Write block 156 len 36864 data: 3 3 2 2 2 2 3 3 4
7422024-02-08T06:39:24.469Z13/30 Read block 294 len 20480
7432024-02-08T06:39:24.469Z14/30 Read block 91 len 40960
7442024-02-08T06:39:24.469Z15/30 Flush
7452024-02-08T06:39:24.469Z16/30 Flush
7462024-02-08T06:39:24.470Z17/30 Write block 130 len 32768 data: 2 2 2 2 2 2 3 3
7472024-02-08T06:39:24.470Z18/30 Write block 542 len 32768 data: 2 2 2 2 2 2 2 2
7482024-02-08T06:39:24.470Z19/30 Write block 588 len 8192 data: 2 2
7492024-02-08T06:39:24.470Z20/30 Read block 239 len 40960
7502024-02-08T06:39:24.470Z21/30 Write block 137 len 32768 data: 4 3 3 3 3 3 2 2
7512024-02-08T06:39:24.471Z22/30 Read block 151 len 4096
7522024-02-08T06:39:24.471Z23/30 Write block 387 len 16384 data: 3 3 3 2
7532024-02-08T06:39:24.471Z24/30 Read block 127 len 8192
7542024-02-08T06:39:24.471Z25/30 Write block 137 len 4096 data: 5
7552024-02-08T06:39:24.471Z26/30 Flush
7562024-02-08T06:39:24.477Z27/30 Write block 430 len 16384 data: 2 2 2 2
7572024-02-08T06:39:24.477Z28/30 Read block 136 len 8192
7582024-02-08T06:39:24.477Z29/30 Write block 416 len 12288 data: 2 2 2
7592024-02-08T06:39:24.478Z30/30 Write block 25 len 16384 data: 3 3 3 2
7602024-02-08T06:39:24.478Z----------------------------------------------------------------
7612024-02-08T06:39:24.478Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:29
7622024-02-08T06:39:24.478ZStates: Active Active Active
7632024-02-08T06:39:24.478ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7642024-02-08T06:39:24.479Z 2 Acked 1001 Write 8 Done Done Done false
7652024-02-08T06:39:24.479Z 3 Acked 1002 Read 6 Done Done Done false
7662024-02-08T06:39:24.479Z 4 Acked 1003 Read 8 Done Done Done false
7672024-02-08T06:39:24.479Z 5 Acked 1004 Write 10 Done Done Done false
7682024-02-08T06:39:24.479Z 6 Acked 1005 Write 3 Done Sent Done false
7692024-02-08T06:39:24.480Z 7 Acked 1006 Read 8 Done Sent Done false
7702024-02-08T06:39:24.480Z 8 Acked 1007 Read 3 Done Sent Done false
7712024-02-08T06:39:24.480Z 9 Acked 1008 Read 10 Done Sent Done false
7722024-02-08T06:39:24.480Z 10 Acked 1009 Write 4 Done Sent Done false
7732024-02-08T06:39:24.480Z 11 Acked 1010 Flush 0 Done Sent Done false
7742024-02-08T06:39:24.481Z 12 Acked 1011 Write 9 Done Sent Done false
7752024-02-08T06:39:24.481Z 13 Acked 1012 Read 5 Done Sent Done false
7762024-02-08T06:39:24.481Z 14 Acked 1013 Read 10 Done Sent Done false
7772024-02-08T06:39:24.481Z 15 Acked 1014 Flush 0 Done Sent Done false
7782024-02-08T06:39:24.481Z 16 Acked 1015 Flush 0 Done Sent Done false
7792024-02-08T06:39:24.482Z 17 Acked 1016 Write 8 Done Sent Done false
7802024-02-08T06:39:24.482Z 18 Acked 1017 Write 8 Done Sent Done false
7812024-02-08T06:39:24.482Z 19 Acked 1018 Write 2 Done Sent Done false
7822024-02-08T06:39:24.482Z 20 Acked 1019 Read 10 Done Sent Done false
7832024-02-08T06:39:24.482Z 21 Acked 1020 Write 8 Done Sent Done false
7842024-02-08T06:39:24.482Z 22 Acked 1021 Read 1 Done Sent Done false
7852024-02-08T06:39:24.483Z 23 Acked 1022 Write 4 Done Sent Done false
7862024-02-08T06:39:24.483Z 24 Acked 1023 Read 2 Done Sent Done false
7872024-02-08T06:39:24.483Z 25 Acked 1024 Write 1 Done Sent Done false
7882024-02-08T06:39:24.483Z 26 Acked 1025 Flush 0 Done Sent Done false
7892024-02-08T06:39:24.483Z 27 Acked 1026 Write 4 Done Sent Done false
7902024-02-08T06:39:24.484Z 28 Acked 1027 Read 2 Done Sent Done false
7912024-02-08T06:39:24.484Z 29 Acked 1028 Write 3 Done Sent Done false
7922024-02-08T06:39:24.484Z 30 Acked 1029 Write 4 Sent Sent Sent false
7932024-02-08T06:39:24.484Z STATES DS:0 DS:1 DS:2 TOTAL
7942024-02-08T06:39:24.485Z New 0 0 0 0
7952024-02-08T06:39:24.485Z Sent 1 25 1 27
7962024-02-08T06:39:24.485Z Done 28 4 28 60
7972024-02-08T06:39:24.485Z Skipped 0 0 0 0
7982024-02-08T06:39:24.485Z Error 0 0 0 0
7992024-02-08T06:39:24.486ZLast Flush: 1025 1000 1025
8002024-02-08T06:39:24.486ZDownstairs last five completed: 1000
8012024-02-08T06:39:24.486ZUpstairs last five completed: 30 29 28 27 26
8022024-02-08T06:39:24.486Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:23.720401385Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1159}
8032024-02-08T06:39:24.486ZWrote out file "/tmp/test_repair_verify.data"
8042024-02-08T06:39:24.486Z
8052024-02-08T06:39:24.487ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8062024-02-08T06:39:24.492ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8072024-02-08T06:39:24.492Z 0 000-019 1 1 1 1 1 1 F F F
8082024-02-08T06:39:24.492Z 1 020-039 6 3 6 10 3 10 T T T <---
8092024-02-08T06:39:24.493Z 2 040-059 6 1 6 10 1 10 F T F <---
8102024-02-08T06:39:24.493Z 3 060-079 1 1 1 1 1 1 F F F
8112024-02-08T06:39:24.493Z 4 080-099 4 4 4 6 6 6 F F F
8122024-02-08T06:39:24.493Z 5 100-119 1 1 1 1 1 1 F F F
8132024-02-08T06:39:24.493Z 6 120-139 6 2 6 13 2 13 F F F <---
8142024-02-08T06:39:24.494Z 7 140-159 6 4 6 13 6 13 F F F <---
8152024-02-08T06:39:24.494Z 8 160-179 6 5 6 11 8 11 F F F <---
8162024-02-08T06:39:24.494Z 9 180-199 5 5 5 8 8 8 F F F
8172024-02-08T06:39:24.494Z 10 200-219 3 3 3 3 3 3 F F F
8182024-02-08T06:39:24.494Z 11 220-239 4 4 4 4 4 4 F F F
8192024-02-08T06:39:24.495Z 12 240-259 2 2 2 2 2 2 F F F
8202024-02-08T06:39:24.495Z 13 260-279 5 5 5 8 8 8 F F F
8212024-02-08T06:39:24.495Z 14 280-299 4 4 4 5 5 5 F F F
8222024-02-08T06:39:24.495Z 15 300-319 4 4 4 4 4 4 F F F
8232024-02-08T06:39:24.496Z 16 320-339 4 4 4 5 5 5 F F F
8242024-02-08T06:39:24.496Z 17 340-359 4 4 4 5 5 5 F F F
8252024-02-08T06:39:24.496Z 18 360-379 3 3 3 3 3 3 F F F
8262024-02-08T06:39:24.496Z 19 380-399 6 4 6 13 7 13 F F F <---
8272024-02-08T06:39:24.496Z 20 400-419 2 2 2 2 2 2 T F T <---
8282024-02-08T06:39:24.497Z 21 420-439 4 4 4 4 4 4 T F T <---
8292024-02-08T06:39:24.497Z 22 440-459 4 4 4 4 4 4 F F F
8302024-02-08T06:39:24.497Z 23 460-479 6 1 6 10 1 10 F F F <---
8312024-02-08T06:39:24.497Z 24 480-499 4 4 4 5 5 5 F F F
8322024-02-08T06:39:24.497Z 25 500-519 1 1 1 1 1 1 F F F
8332024-02-08T06:39:24.497Z 26 520-539 1 1 1 1 1 1 F F F
8342024-02-08T06:39:24.498Z 27 540-559 6 3 6 13 3 13 F F F <---
8352024-02-08T06:39:24.498Z 28 560-579 6 5 6 10 8 10 F T F <---
8362024-02-08T06:39:24.498Z 29 580-599 6 1 6 13 1 13 F F F <---
8372024-02-08T06:39:24.498ZMax gen: 6, Max flush: 13
8382024-02-08T06:39:24.498ZError: Difference in extent metadata found!
8392024-02-08T06:39:24.499ZA Difference in extent metadata is expected here
8402024-02-08T06:39:24.499ZOn loop 3
8412024-02-08T06:39:24.499Z
8422024-02-08T06:39:24.503ZVerifying data now
8432024-02-08T06:39:30.073ZNo change after verify
8442024-02-08T06:39:30.073ZLoop: 3 Downstairs dump after verify (and repair):
8452024-02-08T06:39:30.088ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8462024-02-08T06:39:30.088Z 0 000-019 1 1 1 1 1 1 F F F
8472024-02-08T06:39:30.094Z 1 020-039 7 7 7 14 14 14 F F F
8482024-02-08T06:39:30.094Z 2 040-059 6 6 6 10 10 10 F F F
8492024-02-08T06:39:30.094Z 3 060-079 1 1 1 1 1 1 F F F
8502024-02-08T06:39:30.095Z 4 080-099 4 4 4 6 6 6 F F F
8512024-02-08T06:39:30.095Z 5 100-119 1 1 1 1 1 1 F F F
8522024-02-08T06:39:30.095Z 6 120-139 6 6 6 13 13 13 F F F
8532024-02-08T06:39:30.095Z 7 140-159 6 6 6 13 13 13 F F F
8542024-02-08T06:39:30.095Z 8 160-179 6 6 6 11 11 11 F F F
8552024-02-08T06:39:30.095Z 9 180-199 5 5 5 8 8 8 F F F
8562024-02-08T06:39:30.095Z 10 200-219 3 3 3 3 3 3 F F F
8572024-02-08T06:39:30.095Z 11 220-239 4 4 4 4 4 4 F F F
8582024-02-08T06:39:30.095Z 12 240-259 2 2 2 2 2 2 F F F
8592024-02-08T06:39:30.095Z 13 260-279 5 5 5 8 8 8 F F F
8602024-02-08T06:39:30.095Z 14 280-299 4 4 4 5 5 5 F F F
8612024-02-08T06:39:30.095Z 15 300-319 4 4 4 4 4 4 F F F
8622024-02-08T06:39:30.095Z 16 320-339 4 4 4 5 5 5 F F F
8632024-02-08T06:39:30.095Z 17 340-359 4 4 4 5 5 5 F F F
8642024-02-08T06:39:30.095Z 18 360-379 3 3 3 3 3 3 F F F
8652024-02-08T06:39:30.096Z 19 380-399 6 6 6 13 13 13 F F F
8662024-02-08T06:39:30.096Z 20 400-419 7 7 7 14 14 14 F F F
8672024-02-08T06:39:30.096Z 21 420-439 7 7 7 14 14 14 F F F
8682024-02-08T06:39:30.096Z 22 440-459 4 4 4 4 4 4 F F F
8692024-02-08T06:39:30.096Z 23 460-479 6 6 6 10 10 10 F F F
8702024-02-08T06:39:30.096Z 24 480-499 4 4 4 5 5 5 F F F
8712024-02-08T06:39:30.096Z 25 500-519 1 1 1 1 1 1 F F F
8722024-02-08T06:39:30.096Z 26 520-539 1 1 1 1 1 1 F F F
8732024-02-08T06:39:30.096Z 27 540-559 6 6 6 13 13 13 F F F
8742024-02-08T06:39:30.096Z 28 560-579 6 6 6 10 10 10 F F F
8752024-02-08T06:39:30.096Z 29 580-599 6 6 6 13 13 13 F F F
8762024-02-08T06:39:30.096ZMax gen: 7, Max flush: 14
8772024-02-08T06:39:30.096Z
8782024-02-08T06:39:30.096ZBegin loop 4
8792024-02-08T06:39:30.096ZDownstairs to restart: 1
8802024-02-08T06:39:30.109Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.364731751Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167}
8812024-02-08T06:39:30.114Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365042168Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167}
8822024-02-08T06:39:30.114ZCrucible runtime is spawned
8832024-02-08T06:39:30.115Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365089404Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167}
8842024-02-08T06:39:30.115Z{"msg":"Crucible ab5f9583-b374-4aec-9b4e-8ed9712953fa has session id: 043b3802-f434-4754-99ef-b0d72b7f5ebf","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365115992Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8852024-02-08T06:39:30.115Z{"msg":"Upstairs opts: Upstairs UUID: ab5f9583-b374-4aec-9b4e-8ed9712953fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365165949Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8862024-02-08T06:39:30.115Z{"msg":"Crucible stats registered with UUID: ab5f9583-b374-4aec-9b4e-8ed9712953fa","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365193136Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8872024-02-08T06:39:30.115Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365224854Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167}
8882024-02-08T06:39:30.115Z{"msg":"ab5f9583-b374-4aec-9b4e-8ed9712953fa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365246292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8892024-02-08T06:39:30.115Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.36527457Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8902024-02-08T06:39:30.115Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365320827Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8912024-02-08T06:39:30.115Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365345385Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8922024-02-08T06:39:30.115Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365398471Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8932024-02-08T06:39:30.115Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365486025Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8942024-02-08T06:39:30.115Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365530821Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8952024-02-08T06:39:30.115Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365566379Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8962024-02-08T06:39:30.115Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365600676Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8972024-02-08T06:39:30.115Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365624504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8982024-02-08T06:39:30.116Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:29.365664991Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
8992024-02-08T06:39:30.116Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:29.365706178Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9002024-02-08T06:39:30.116Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:29.365736706Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9012024-02-08T06:39:30.116Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365760624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9022024-02-08T06:39:30.116Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365805061Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9032024-02-08T06:39:30.116Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365833808Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9042024-02-08T06:39:30.116Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365863906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9052024-02-08T06:39:30.116Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.365885335Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9062024-02-08T06:39:30.116Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.366365328Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9072024-02-08T06:39:30.116Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.366406945Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9082024-02-08T06:39:30.116Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.366423684Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9092024-02-08T06:39:30.116Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.366437873Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9102024-02-08T06:39:30.116Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.36700941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9112024-02-08T06:39:30.116Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.367058027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9122024-02-08T06:39:30.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.367652922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9132024-02-08T06:39:30.117Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.367691999Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9142024-02-08T06:39:30.117Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.367725556Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9152024-02-08T06:39:30.117Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.367741515Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9162024-02-08T06:39:30.117Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.367762954Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9172024-02-08T06:39:30.117Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:29.367777593Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9182024-02-08T06:39:31.112Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.367231206Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9192024-02-08T06:39:31.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.367375555Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9202024-02-08T06:39:31.117Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.368388409Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9212024-02-08T06:39:31.117Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.368426046Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9222024-02-08T06:39:31.117Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.368915549Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9232024-02-08T06:39:31.117Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369448129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9242024-02-08T06:39:31.117Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369491526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9252024-02-08T06:39:31.117Z{"msg":"[0]R flush_numbers[0..12]: [1, 14, 10, 1, 6, 1, 13, 13, 11, 8, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369532043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9262024-02-08T06:39:31.118Z{"msg":"[0]R generation[0..12]: [1, 7, 6, 1, 4, 1, 6, 6, 6, 5, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.36956439Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9272024-02-08T06:39:31.118Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369583829Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9282024-02-08T06:39:31.118Z{"msg":"[1]R flush_numbers[0..12]: [1, 14, 10, 1, 6, 1, 13, 13, 11, 8, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369609487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9292024-02-08T06:39:31.118Z{"msg":"[1]R generation[0..12]: [1, 7, 6, 1, 4, 1, 6, 6, 6, 5, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369626816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9302024-02-08T06:39:31.118Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369645255Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9312024-02-08T06:39:31.118Z{"msg":"[2]R flush_numbers[0..12]: [1, 14, 10, 1, 6, 1, 13, 13, 11, 8, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369662623Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9322024-02-08T06:39:31.118Z{"msg":"[2]R generation[0..12]: [1, 7, 6, 1, 4, 1, 6, 6, 6, 5, 3, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369695081Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9332024-02-08T06:39:31.118ZWait for a query_work_queue command to finish before sending IO
9342024-02-08T06:39:31.118Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369731418Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9352024-02-08T06:39:31.118Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369757056Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9362024-02-08T06:39:31.118Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369775325Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9372024-02-08T06:39:31.118Z{"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369808362Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9382024-02-08T06:39:31.118Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.36984446Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9392024-02-08T06:39:31.118Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369862428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9402024-02-08T06:39:31.119Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369894466Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9412024-02-08T06:39:31.119Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369922664Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9422024-02-08T06:39:31.119Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369956911Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9432024-02-08T06:39:31.119Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.369986659Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9442024-02-08T06:39:31.119Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.370016417Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9452024-02-08T06:39:31.119Z{"msg":"ab5f9583-b374-4aec-9b4e-8ed9712953fa is now active with session: 043b3802-f434-4754-99ef-b0d72b7f5ebf","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.370031495Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9462024-02-08T06:39:31.119Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.370061473Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167,"session_id":"043b3802-f434-4754-99ef-b0d72b7f5ebf"}
9472024-02-08T06:39:31.119ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9482024-02-08T06:39:31.119Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:30.370092001Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167}
9492024-02-08T06:39:31.119ZLoading write count information from file "/tmp/test_repair_verify.data"
9502024-02-08T06:39:31.119ZRun Repair workload
9512024-02-08T06:39:31.119Z01/30 Write block 173 len 4096 data: 3
9522024-02-08T06:39:31.123Z02/30 Read block 206 len 40960
9532024-02-08T06:39:31.123Z03/30 Write block 131 len 4096 data: 3
9542024-02-08T06:39:31.123Z04/30 Write block 221 len 32768 data: 2 2 2 2 3 3 3 3
9552024-02-08T06:39:31.123Z05/30 Write block 350 len 32768 data: 3 3 2 2 2 2 2 2
9562024-02-08T06:39:31.123Z06/30 Read block 214 len 12288
9572024-02-08T06:39:31.123Z07/30 Read block 174 len 36864
9582024-02-08T06:39:31.124Z08/30 Read block 585 len 4096
9592024-02-08T06:39:31.124Z09/30 Write block 556 len 32768 data: 3 3 3 3 3 3 3 3
9602024-02-08T06:39:31.124Z10/30 Write block 181 len 12288 data: 2 2 2
9612024-02-08T06:39:31.129Z11/30 Write block 462 len 8192 data: 2 2
9622024-02-08T06:39:31.130Z12/30 Write block 295 len 20480 data: 2 2 2 2 4
9632024-02-08T06:39:31.130Z13/30 Read block 509 len 12288
9642024-02-08T06:39:31.130Z14/30 Write block 290 len 8192 data: 3 3
9652024-02-08T06:39:31.130Z15/30 Write block 336 len 36864 data: 3 3 3 3 3 3 3 2 2
9662024-02-08T06:39:31.130Z16/30 Write block 349 len 36864 data: 3 4 4 3 3 3 3 3 3
9672024-02-08T06:39:31.130Z17/30 Write block 82 len 24576 data: 2 2 2 2 2 2
9682024-02-08T06:39:31.130Z18/30 Write block 178 len 8192 data: 2 2
9692024-02-08T06:39:31.133Z19/30 Write block 518 len 12288 data: 2 2 2
9702024-02-08T06:39:31.133Z20/30 Write block 274 len 8192 data: 2 2
9712024-02-08T06:39:31.134Z21/30 Write block 287 len 40960 data: 4 3 3 4 4 3 2 2 3 3
9722024-02-08T06:39:31.134Z22/30 Write block 3 len 32768 data: 2 2 2 2 2 2 2 2
9732024-02-08T06:39:31.134Z23/30 Read block 338 len 12288
9742024-02-08T06:39:31.141Z24/30 Read block 363 len 40960
9752024-02-08T06:39:31.141Z25/30 Write block 228 len 20480 data: 4 3 2 2 2
9762024-02-08T06:39:31.141Z26/30 Write block 484 len 36864 data: 2 2 2 2 2 2 2 2 2
9772024-02-08T06:39:31.142Z27/30 Write block 103 len 28672 data: 2 2 2 2 2 2 2
9782024-02-08T06:39:31.146Z28/30 Write block 277 len 4096 data: 2
9792024-02-08T06:39:31.146Z29/30 Read block 552 len 32768
9802024-02-08T06:39:31.147Z30/30 Write block 98 len 28672 data: 2 2 2 2 2 3 3
9812024-02-08T06:39:31.147Z----------------------------------------------------------------
9822024-02-08T06:39:31.147Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9832024-02-08T06:39:31.147ZStates: Active Active Active
9842024-02-08T06:39:31.147ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9852024-02-08T06:39:31.147Z 1 Acked 1000 Write 1 Done Done Done false
9862024-02-08T06:39:31.147Z 2 Acked 1001 Read 10 Done Sent Done false
9872024-02-08T06:39:31.148Z 3 Acked 1002 Write 1 Done Sent Done false
9882024-02-08T06:39:31.148Z 4 Acked 1003 Write 8 Done Sent Done false
9892024-02-08T06:39:31.148Z 5 Acked 1004 Write 8 Done Sent Done false
9902024-02-08T06:39:31.148Z 6 Acked 1005 Read 3 Done Sent Done false
9912024-02-08T06:39:31.148Z 7 Acked 1006 Read 9 Done Sent Done false
9922024-02-08T06:39:31.148Z 8 Acked 1007 Read 1 Done Sent Done false
9932024-02-08T06:39:31.148Z 9 Acked 1008 Write 8 Done Sent Done false
9942024-02-08T06:39:31.148Z 10 Acked 1009 Write 3 Done Sent Done false
9952024-02-08T06:39:31.148Z 11 Acked 1010 Write 2 Done Sent Done false
9962024-02-08T06:39:31.148Z 12 Acked 1011 Write 5 Done Sent Done false
9972024-02-08T06:39:31.148Z 13 Acked 1012 Read 3 Done Sent Done false
9982024-02-08T06:39:31.148Z 14 Acked 1013 Write 2 Done Sent Done false
9992024-02-08T06:39:31.149Z 15 Acked 1014 Write 9 Done Sent Done false
10002024-02-08T06:39:31.149Z 16 Acked 1015 Write 9 Done Sent Done false
10012024-02-08T06:39:31.149Z 17 Acked 1016 Write 6 Done Sent Done false
10022024-02-08T06:39:31.149Z 18 Acked 1017 Write 2 Done Sent Done false
10032024-02-08T06:39:31.149Z 19 Acked 1018 Write 3 Done Sent Done false
10042024-02-08T06:39:31.149Z 20 Acked 1019 Write 2 Done Sent Done false
10052024-02-08T06:39:31.149Z 21 Acked 1020 Write 10 Done Sent Done false
10062024-02-08T06:39:31.149Z 22 Acked 1021 Write 8 Done Sent Done false
10072024-02-08T06:39:31.149Z 23 Acked 1022 Read 3 Done Sent Done false
10082024-02-08T06:39:31.149Z 24 Acked 1023 Read 10 Done Sent Done false
10092024-02-08T06:39:31.149Z 25 Acked 1024 Write 5 Done Sent Done false
10102024-02-08T06:39:31.149Z 26 Acked 1025 Write 9 Done Sent Done false
10112024-02-08T06:39:31.149Z 27 Acked 1026 Write 7 Done Sent Done false
10122024-02-08T06:39:31.149Z 28 Acked 1027 Write 1 Done Sent Done false
10132024-02-08T06:39:31.149Z 29 Acked 1028 Read 8 Done Sent Done false
10142024-02-08T06:39:31.149Z 30 Acked 1029 Write 7 Sent Sent Sent false
10152024-02-08T06:39:31.150Z STATES DS:0 DS:1 DS:2 TOTAL
10162024-02-08T06:39:31.150Z New 0 0 0 0
10172024-02-08T06:39:31.150Z Sent 1 29 1 31
10182024-02-08T06:39:31.150Z Done 29 1 29 59
10192024-02-08T06:39:31.150Z Skipped 0 0 0 0
10202024-02-08T06:39:31.150Z Error 0 0 0 0
10212024-02-08T06:39:31.150ZLast Flush: 0 0 0
10222024-02-08T06:39:31.150ZDownstairs last five completed:
10232024-02-08T06:39:31.150ZUpstairs last five completed: 30 29 28 27 26
10242024-02-08T06:39:31.150Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:30.395533081Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1167}
10252024-02-08T06:39:31.150ZWrote out file "/tmp/test_repair_verify.data"
10262024-02-08T06:39:31.150Z
10272024-02-08T06:39:31.159ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10282024-02-08T06:39:31.178ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10292024-02-08T06:39:31.178Z 0 000-019 1 1 1 1 1 1 T F T <---
10302024-02-08T06:39:31.178Z 1 020-039 7 7 7 14 14 14 F F F
10312024-02-08T06:39:31.178Z 2 040-059 6 6 6 10 10 10 F F F
10322024-02-08T06:39:31.702Z 3 060-079 1 1 1 1 1 1 F F F
10332024-02-08T06:39:31.702Z 4 080-099 4 4 4 6 6 6 T F T <---
10342024-02-08T06:39:31.702Z 5 100-119 1 1 1 1 1 1 T F T <---
10352024-02-08T06:39:31.702Z 6 120-139 6 6 6 13 13 13 T F T <---
10362024-02-08T06:39:31.702Z 7 140-159 6 6 6 13 13 13 F F F
10372024-02-08T06:39:31.702Z 8 160-179 6 6 6 11 11 11 T T T
10382024-02-08T06:39:31.702Z 9 180-199 5 5 5 8 8 8 T F T <---
10392024-02-08T06:39:31.703Z 10 200-219 3 3 3 3 3 3 F F F
10402024-02-08T06:39:31.703Z 11 220-239 4 4 4 4 4 4 T F T <---
10412024-02-08T06:39:31.703Z 12 240-259 2 2 2 2 2 2 F F F
10422024-02-08T06:39:31.703Z 13 260-279 5 5 5 8 8 8 T F T <---
10432024-02-08T06:39:31.703Z 14 280-299 4 4 4 5 5 5 T F T <---
10442024-02-08T06:39:31.703Z 15 300-319 4 4 4 4 4 4 F F F
10452024-02-08T06:39:31.703Z 16 320-339 4 4 4 5 5 5 T F T <---
10462024-02-08T06:39:31.703Z 17 340-359 4 4 4 5 5 5 T F T <---
10472024-02-08T06:39:31.703Z 18 360-379 3 3 3 3 3 3 F F F
10482024-02-08T06:39:31.703Z 19 380-399 6 6 6 13 13 13 F F F
10492024-02-08T06:39:31.703Z 20 400-419 7 7 7 14 14 14 F F F
10502024-02-08T06:39:31.703Z 21 420-439 7 7 7 14 14 14 F F F
10512024-02-08T06:39:31.703Z 22 440-459 4 4 4 4 4 4 F F F
10522024-02-08T06:39:31.703Z 23 460-479 6 6 6 10 10 10 T F T <---
10532024-02-08T06:39:31.703Z 24 480-499 4 4 4 5 5 5 T F T <---
10542024-02-08T06:39:31.704Z 25 500-519 1 1 1 1 1 1 T F T <---
10552024-02-08T06:39:31.704Z 26 520-539 1 1 1 1 1 1 T F T <---
10562024-02-08T06:39:31.704Z 27 540-559 6 6 6 13 13 13 T F T <---
10572024-02-08T06:39:31.704Z 28 560-579 6 6 6 10 10 10 T F T <---
10582024-02-08T06:39:31.704Z 29 580-599 6 6 6 13 13 13 F F F
10592024-02-08T06:39:31.704ZMax gen: 7, Max flush: 14
10602024-02-08T06:39:31.704ZError: Difference in extent metadata found!
10612024-02-08T06:39:31.704ZA Difference in extent metadata is expected here
10622024-02-08T06:39:31.704ZOn loop 4
10632024-02-08T06:39:31.704Z
10642024-02-08T06:39:31.704ZVerifying data now
10652024-02-08T06:39:35.985ZNo change after verify
10662024-02-08T06:39:35.988ZLoop: 4 Downstairs dump after verify (and repair):
10672024-02-08T06:39:35.999ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10682024-02-08T06:39:35.999Z 0 000-019 8 8 8 15 15 15 F F F
10692024-02-08T06:39:35.999Z 1 020-039 7 7 7 14 14 14 F F F
10702024-02-08T06:39:35.999Z 2 040-059 6 6 6 10 10 10 F F F
10712024-02-08T06:39:35.999Z 3 060-079 1 1 1 1 1 1 F F F
10722024-02-08T06:39:36.000Z 4 080-099 8 8 8 15 15 15 F F F
10732024-02-08T06:39:36.003Z 5 100-119 8 8 8 15 15 15 F F F
10742024-02-08T06:39:36.003Z 6 120-139 8 8 8 15 15 15 F F F
10752024-02-08T06:39:36.003Z 7 140-159 6 6 6 13 13 13 F F F
10762024-02-08T06:39:36.003Z 8 160-179 8 8 8 15 15 15 F F F
10772024-02-08T06:39:36.003Z 9 180-199 8 8 8 15 15 15 F F F
10782024-02-08T06:39:36.003Z 10 200-219 3 3 3 3 3 3 F F F
10792024-02-08T06:39:36.003Z 11 220-239 8 8 8 15 15 15 F F F
10802024-02-08T06:39:36.004Z 12 240-259 2 2 2 2 2 2 F F F
10812024-02-08T06:39:36.004Z 13 260-279 8 8 8 15 15 15 F F F
10822024-02-08T06:39:36.004Z 14 280-299 8 8 8 15 15 15 F F F
10832024-02-08T06:39:36.004Z 15 300-319 4 4 4 4 4 4 F F F
10842024-02-08T06:39:36.004Z 16 320-339 8 8 8 15 15 15 F F F
10852024-02-08T06:39:36.004Z 17 340-359 8 8 8 15 15 15 F F F
10862024-02-08T06:39:36.004Z 18 360-379 3 3 3 3 3 3 F F F
10872024-02-08T06:39:36.004Z 19 380-399 6 6 6 13 13 13 F F F
10882024-02-08T06:39:36.004Z 20 400-419 7 7 7 14 14 14 F F F
10892024-02-08T06:39:36.004Z 21 420-439 7 7 7 14 14 14 F F F
10902024-02-08T06:39:36.004Z 22 440-459 4 4 4 4 4 4 F F F
10912024-02-08T06:39:36.004Z 23 460-479 8 8 8 15 15 15 F F F
10922024-02-08T06:39:36.004Z 24 480-499 8 8 8 15 15 15 F F F
10932024-02-08T06:39:36.004Z 25 500-519 8 8 8 15 15 15 F F F
10942024-02-08T06:39:36.004Z 26 520-539 8 8 8 15 15 15 F F F
10952024-02-08T06:39:36.005Z 27 540-559 8 8 8 15 15 15 F F F
10962024-02-08T06:39:36.005Z 28 560-579 8 8 8 15 15 15 F F F
10972024-02-08T06:39:36.005Z 29 580-599 6 6 6 13 13 13 F F F
10982024-02-08T06:39:36.005ZMax gen: 8, Max flush: 15
10992024-02-08T06:39:36.005Z
11002024-02-08T06:39:36.005ZBegin loop 5
11012024-02-08T06:39:36.005ZDownstairs to restart: 1
11022024-02-08T06:39:36.020Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.274864152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175}
11032024-02-08T06:39:36.020Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"Crucible runtime is spawned
11042024-02-08T06:39:36.025Z1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275202757Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175}
11052024-02-08T06:39:36.025Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275266092Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175}
11062024-02-08T06:39:36.025Z{"msg":"Crucible 33fdeb09-354e-4fcf-870b-f0f5c4988d36 has session id: befeb050-25c9-4dcf-bca3-39dce1c7f321","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.27529518Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11072024-02-08T06:39:36.025Z{"msg":"Upstairs opts: Upstairs UUID: 33fdeb09-354e-4fcf-870b-f0f5c4988d36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275350726Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11082024-02-08T06:39:36.025Z{"msg":"Crucible stats registered with UUID: 33fdeb09-354e-4fcf-870b-f0f5c4988d36","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275396612Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11092024-02-08T06:39:36.026Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275438449Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175}
11102024-02-08T06:39:36.026Z{"msg":"33fdeb09-354e-4fcf-870b-f0f5c4988d36 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275479736Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11112024-02-08T06:39:36.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275513903Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11122024-02-08T06:39:36.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275542521Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11132024-02-08T06:39:36.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275620695Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11142024-02-08T06:39:36.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275657203Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11152024-02-08T06:39:36.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.27569532Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11162024-02-08T06:39:36.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275723698Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11172024-02-08T06:39:36.026Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275770814Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11182024-02-08T06:39:36.026Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275811641Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11192024-02-08T06:39:36.026Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275885455Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11202024-02-08T06:39:36.026Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275919393Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11212024-02-08T06:39:36.026Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275944061Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11222024-02-08T06:39:36.026Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.275968109Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11232024-02-08T06:39:36.026Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.276706974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11242024-02-08T06:39:36.027Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.276764149Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11252024-02-08T06:39:36.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.276843663Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11262024-02-08T06:39:36.027Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.27689116Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11272024-02-08T06:39:36.027Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.277316458Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11282024-02-08T06:39:36.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.277455528Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11292024-02-08T06:39:36.027Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.277492365Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11302024-02-08T06:39:36.027Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.277509673Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11312024-02-08T06:39:36.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.277885635Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11322024-02-08T06:39:36.027Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.277923062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11332024-02-08T06:39:36.027Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.277939751Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11342024-02-08T06:39:36.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278095469Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11352024-02-08T06:39:36.027Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278123667Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11362024-02-08T06:39:36.027Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278138316Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11372024-02-08T06:39:36.027Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278170714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11382024-02-08T06:39:36.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278638889Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11392024-02-08T06:39:36.027Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278671416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11402024-02-08T06:39:36.033Z{"msg":"[0]R flush_numbers[0..12]: [15, 14, 10, 1, 15, 15, 15, 13, 15, 15, 3, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278690325Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11412024-02-08T06:39:36.033Z{"msg":"[0]R generation[0..12]: [8, 7, 6, 1, 8, 8, 8, 6, 8, 8, 3, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278723002Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11422024-02-08T06:39:36.033Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.27874936Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11432024-02-08T06:39:36.033Z{"msg":"[1]R flush_numbers[0..12]: [15, 14, 10, 1, 15, 15, 15, 13, 15, 15, 3, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278782568Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11442024-02-08T06:39:36.033Z{"msg":"[1]R generation[0..12]: [8, 7, 6, 1, 8, 8, 8, 6, 8, 8, 3, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278808166Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11452024-02-08T06:39:36.033ZWait for a query_work_queue command to finish before sending IO
11462024-02-08T06:39:36.033Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278840824Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11472024-02-08T06:39:36.033Z{"msg":"[2]R flush_numbers[0..12]: [15, 14, 10, 1, 15, 15, 15, 13, 15, 15, 3, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278866602Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11482024-02-08T06:39:36.033Z{"msg":"[2]R generation[0..12]: [8, 7, 6, 1, 8, 8, 8, 6, 8, 8, 3, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278906319Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11492024-02-08T06:39:36.033Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278943336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11502024-02-08T06:39:36.033Z{"msg":"Max found gen is 9","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278975943Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11512024-02-08T06:39:36.033Z{"msg":"Generation requested: 10 >= found:9","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.278992352Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11522024-02-08T06:39:36.034Z{"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279036049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11532024-02-08T06:39:36.034Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279058237Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11542024-02-08T06:39:36.034Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279079206Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11552024-02-08T06:39:36.034Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279108443Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11562024-02-08T06:39:36.034Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279123932Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11572024-02-08T06:39:36.034Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279143941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11582024-02-08T06:39:36.034Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279162789Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11592024-02-08T06:39:36.034Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279194157Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11602024-02-08T06:39:36.034Z{"msg":"33fdeb09-354e-4fcf-870b-f0f5c4988d36 is now active with session: befeb050-25c9-4dcf-bca3-39dce1c7f321","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279226355Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11612024-02-08T06:39:36.034Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279256212Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175,"session_id":"befeb050-25c9-4dcf-bca3-39dce1c7f321"}
11622024-02-08T06:39:36.034Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:35.279271041Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175}
11632024-02-08T06:39:36.034ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11642024-02-08T06:39:36.034ZLoading write count information from file "/tmp/test_repair_verify.data"
11652024-02-08T06:39:36.034ZRun Repair workload
11662024-02-08T06:39:36.034Z01/30 Write block 197 len 24576 data: 2 2 2 2 2 2
11672024-02-08T06:39:36.035Z02/30 Read block 215 len 28672
11682024-02-08T06:39:36.035Z03/30 Read block 136 len 16384
11692024-02-08T06:39:36.035Z04/30 Read block 318 len 20480
11702024-02-08T06:39:36.035Z05/30 Write block 456 len 32768 data: 3 3 3 2 2 2 3 3
11712024-02-08T06:39:36.035Z06/30 Read block 113 len 16384
11722024-02-08T06:39:36.035Z07/30 Read block 300 len 16384
11732024-02-08T06:39:36.038Z08/30 Read block 347 len 32768
11742024-02-08T06:39:36.039Z09/30 Read block 175 len 4096
11752024-02-08T06:39:36.039Z10/30 Read block 465 len 8192
11762024-02-08T06:39:36.039Z11/30 Read block 440 len 36864
11772024-02-08T06:39:36.039Z12/30 Read block 31 len 40960
11782024-02-08T06:39:36.039Z13/30 Flush
11792024-02-08T06:39:36.039Z14/30 Write block 562 len 4096 data: 4
11802024-02-08T06:39:36.039Z15/30 Write block 224 len 4096 data: 3
11812024-02-08T06:39:36.042Z16/30 Flush
11822024-02-08T06:39:36.042Z17/30 Write block 150 len 8192 data: 4 4
11832024-02-08T06:39:36.042Z18/30 Read block 288 len 8192
11842024-02-08T06:39:36.042Z19/30 Write block 276 len 32768 data: 2 3 2 2 2 2 2 2
11852024-02-08T06:39:36.046Z20/30 Write block 570 len 24576 data: 2 2 2 2 2 2
11862024-02-08T06:39:36.046Z21/30 Read block 302 len 36864
11872024-02-08T06:39:36.046Z22/30 Write block 345 len 12288 data: 3 3 3
11882024-02-08T06:39:36.046Z23/30 Write block 91 len 20480 data: 2 3 3 3 3
11892024-02-08T06:39:36.046Z24/30 Flush
11902024-02-08T06:39:36.049Z25/30 Flush
11912024-02-08T06:39:36.049Z26/30 Write block 504 len 28672 data: 2 2 2 2 2 2 2
11922024-02-08T06:39:36.054Z27/30 Read block 108 len 36864
11932024-02-08T06:39:36.054Z28/30 Write block 198 len 36864 data: 3 3 3 3 3 2 2 2 2
11942024-02-08T06:39:36.054Z29/30 Write block 33 len 4096 data: 2
11952024-02-08T06:39:36.054Z30/30 Write block 469 len 4096 data: 2
11962024-02-08T06:39:36.057Z----------------------------------------------------------------
11972024-02-08T06:39:36.057Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11982024-02-08T06:39:36.057ZStates: Active Active Active
11992024-02-08T06:39:36.057ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
12002024-02-08T06:39:36.057Z 1 Acked 1000 Write 6 Done Done Done false
12012024-02-08T06:39:36.058Z 2 Acked 1001 Read 7 Done Done Done false
12022024-02-08T06:39:36.058Z 3 Acked 1002 Read 4 Done Sent Done false
12032024-02-08T06:39:36.058Z 4 Acked 1003 Read 5 Done Sent Done false
12042024-02-08T06:39:36.058Z 5 Acked 1004 Write 8 Done Sent Done false
12052024-02-08T06:39:36.058Z 6 Acked 1005 Read 4 Done Sent Done false
12062024-02-08T06:39:36.058Z 7 Acked 1006 Read 4 Done Sent Done false
12072024-02-08T06:39:36.058Z 8 Acked 1007 Read 8 Done Sent Done false
12082024-02-08T06:39:36.058Z 9 Acked 1008 Read 1 Done Sent Done false
12092024-02-08T06:39:36.058Z 10 Acked 1009 Read 2 Done Sent Done false
12102024-02-08T06:39:36.058Z 11 Acked 1010 Read 9 Done Sent Done false
12112024-02-08T06:39:36.058Z 12 Acked 1011 Read 10 Done Sent Done false
12122024-02-08T06:39:36.058Z 13 Acked 1012 Flush 0 Done Sent Done false
12132024-02-08T06:39:36.058Z 14 Acked 1013 Write 1 Done Sent Done false
12142024-02-08T06:39:36.058Z 15 Acked 1014 Write 1 Done Sent Done false
12152024-02-08T06:39:36.058Z 16 Acked 1015 Flush 0 Done Sent Done false
12162024-02-08T06:39:36.058Z 17 Acked 1016 Write 2 Done Sent Done false
12172024-02-08T06:39:36.059Z 18 Acked 1017 Read 2 Done Sent Done false
12182024-02-08T06:39:36.059Z 19 Acked 1018 Write 8 Done Sent Done false
12192024-02-08T06:39:36.059Z 20 Acked 1019 Write 6 Done Sent Done false
12202024-02-08T06:39:36.059Z 21 Acked 1020 Read 9 Done Sent Done false
12212024-02-08T06:39:36.059Z 22 Acked 1021 Write 3 Done Sent Done false
12222024-02-08T06:39:36.059Z 23 Acked 1022 Write 5 Done Sent Done false
12232024-02-08T06:39:36.059Z 24 Acked 1023 Flush 0 Done Sent Done false
12242024-02-08T06:39:36.059Z 25 Acked 1024 Flush 0 Done Sent Done false
12252024-02-08T06:39:36.059Z 26 Acked 1025 Write 7 Done Sent Done false
12262024-02-08T06:39:36.059Z 27 Acked 1026 Read 9 Done Sent Done false
12272024-02-08T06:39:36.059Z 28 Acked 1027 Write 9 Sent Sent Sent false
12282024-02-08T06:39:36.059Z 29 Acked 1028 Write 1 Sent Sent Sent false
12292024-02-08T06:39:36.059Z 30 Acked 1029 Write 1 Sent Sent Sent false
12302024-02-08T06:39:36.059Z STATES DS:0 DS:1 DS:2 TOTAL
12312024-02-08T06:39:36.059Z New 0 0 0 0
12322024-02-08T06:39:36.059Z Sent 3 28 3 34
12332024-02-08T06:39:36.060Z Done 27 2 27 56
12342024-02-08T06:39:36.060Z Skipped 0 0 0 0
12352024-02-08T06:39:36.060Z Error 0 0 0 0
12362024-02-08T06:39:36.060ZLast Flush: 1024 0 1024
12372024-02-08T06:39:36.060ZDownstairs last five completed:
12382024-02-08T06:39:36.060ZUpstairs last five completed: 30 29 28 27 26
12392024-02-08T06:39:36.060Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:35.308065179Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1175}
12402024-02-08T06:39:36.060ZWrote out file "/tmp/test_repair_verify.data"
12412024-02-08T06:39:36.060Z
12422024-02-08T06:39:36.064ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12432024-02-08T06:39:36.077ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12442024-02-08T06:39:36.077Z 0 000-019 8 8 8 15 15 15 F F F
12452024-02-08T06:39:36.077Z 1 020-039 7 7 7 14 14 14 T F T <---
12462024-02-08T06:39:36.077Z 2 040-059 6 6 6 10 10 10 F F F
12472024-02-08T06:39:36.077Z 3 060-079 1 1 1 1 1 1 F F F
12482024-02-08T06:39:36.077Z 4 080-099 10 8 10 18 15 18 F F F <---
12492024-02-08T06:39:36.077Z 5 100-119 8 8 8 15 15 15 F F F
12502024-02-08T06:39:36.077Z 6 120-139 8 8 8 15 15 15 F F F
12512024-02-08T06:39:36.077Z 7 140-159 10 6 10 18 13 18 F F F <---
12522024-02-08T06:39:36.077Z 8 160-179 8 8 8 15 15 15 F F F
12532024-02-08T06:39:36.080Z 9 180-199 10 8 10 16 15 16 T T T <---
12542024-02-08T06:39:36.081Z 10 200-219 10 3 10 16 3 16 T T T <---
12552024-02-08T06:39:36.081Z 11 220-239 10 8 10 17 15 17 F F F <---
12562024-02-08T06:39:36.081Z 12 240-259 2 2 2 2 2 2 F F F
12572024-02-08T06:39:36.081Z 13 260-279 10 8 10 18 15 18 F F F <---
12582024-02-08T06:39:36.081Z 14 280-299 10 8 10 18 15 18 F F F <---
12592024-02-08T06:39:36.081Z 15 300-319 4 4 4 4 4 4 F F F
12602024-02-08T06:39:36.081Z 16 320-339 8 8 8 15 15 15 F F F
12612024-02-08T06:39:36.081Z 17 340-359 10 8 10 18 15 18 F F F <---
12622024-02-08T06:39:36.081Z 18 360-379 3 3 3 3 3 3 F F F
12632024-02-08T06:39:36.081Z 19 380-399 6 6 6 13 13 13 F F F
12642024-02-08T06:39:36.081Z 20 400-419 7 7 7 14 14 14 F F F
12652024-02-08T06:39:36.081Z 21 420-439 7 7 7 14 14 14 F F F
12662024-02-08T06:39:36.081Z 22 440-459 10 4 10 16 4 16 F F F <---
12672024-02-08T06:39:36.082Z 23 460-479 10 8 10 16 15 16 T F T <---
12682024-02-08T06:39:36.082Z 24 480-499 8 8 8 15 15 15 F F F
12692024-02-08T06:39:36.082Z 25 500-519 8 8 8 15 15 15 T F T <---
12702024-02-08T06:39:36.082Z 26 520-539 8 8 8 15 15 15 F F F
12712024-02-08T06:39:36.082Z 27 540-559 8 8 8 15 15 15 F F F
12722024-02-08T06:39:36.082Z 28 560-579 10 8 10 18 15 18 F F F <---
12732024-02-08T06:39:36.082Z 29 580-599 6 6 6 13 13 13 F F F
12742024-02-08T06:39:36.082ZMax gen: 10, Max flush: 18
12752024-02-08T06:39:36.082ZError: Difference in extent metadata found!
12762024-02-08T06:39:36.082ZA Difference in extent metadata is expected here
12772024-02-08T06:39:36.082ZOn loop 5
12782024-02-08T06:39:36.082Z
12792024-02-08T06:39:36.086ZVerifying data now
12802024-02-08T06:39:41.700ZNo change after verify
12812024-02-08T06:39:41.700ZLoop: 5 Downstairs dump after verify (and repair):
12822024-02-08T06:39:41.726ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12832024-02-08T06:39:41.726Z 0 000-019 8 8 8 15 15 15 F F F
12842024-02-08T06:39:41.726Z 1 020-039 11 11 11 19 19 19 F F F
12852024-02-08T06:39:41.726Z 2 040-059 6 6 6 10 10 10 F F F
12862024-02-08T06:39:41.726Z 3 060-079 1 1 1 1 1 1 F F F
12872024-02-08T06:39:41.726Z 4 080-099 10 10 10 18 18 18 F F F
12882024-02-08T06:39:41.731Z 5 100-119 8 8 8 15 15 15 F F F
12892024-02-08T06:39:41.731Z 6 120-139 8 8 8 15 15 15 F F F
12902024-02-08T06:39:41.731Z 7 140-159 10 10 10 18 18 18 F F F
12912024-02-08T06:39:41.731Z 8 160-179 8 8 8 15 15 15 F F F
12922024-02-08T06:39:41.731Z 9 180-199 11 11 11 19 19 19 F F F
12932024-02-08T06:39:41.731Z 10 200-219 11 11 11 19 19 19 F F F
12942024-02-08T06:39:41.732Z 11 220-239 10 10 10 17 17 17 F F F
12952024-02-08T06:39:41.732Z 12 240-259 2 2 2 2 2 2 F F F
12962024-02-08T06:39:41.732Z 13 260-279 10 10 10 18 18 18 F F F
12972024-02-08T06:39:41.732Z 14 280-299 10 10 10 18 18 18 F F F
12982024-02-08T06:39:41.732Z 15 300-319 4 4 4 4 4 4 F F F
12992024-02-08T06:39:41.732Z 16 320-339 8 8 8 15 15 15 F F F
13002024-02-08T06:39:41.732Z 17 340-359 10 10 10 18 18 18 F F F
13012024-02-08T06:39:41.732Z 18 360-379 3 3 3 3 3 3 F F F
13022024-02-08T06:39:41.732Z 19 380-399 6 6 6 13 13 13 F F F
13032024-02-08T06:39:41.733Z 20 400-419 7 7 7 14 14 14 F F F
13042024-02-08T06:39:41.733Z 21 420-439 7 7 7 14 14 14 F F F
13052024-02-08T06:39:41.733Z 22 440-459 10 10 10 16 16 16 F F F
13062024-02-08T06:39:41.733Z 23 460-479 11 11 11 19 19 19 F F F
13072024-02-08T06:39:41.733Z 24 480-499 8 8 8 15 15 15 F F F
13082024-02-08T06:39:41.733Z 25 500-519 11 11 11 19 19 19 F F F
13092024-02-08T06:39:41.733Z 26 520-539 8 8 8 15 15 15 F F F
13102024-02-08T06:39:41.733Z 27 540-559 8 8 8 15 15 15 F F F
13112024-02-08T06:39:41.733Z 28 560-579 10 10 10 18 18 18 F F F
13122024-02-08T06:39:41.734Z 29 580-599 6 6 6 13 13 13 F F F
13132024-02-08T06:39:41.734ZMax gen: 11, Max flush: 19
13142024-02-08T06:39:41.734Z
13152024-02-08T06:39:41.734ZBegin loop 6
13162024-02-08T06:39:41.734ZDownstairs to restart: 1
13172024-02-08T06:39:41.741Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.988585049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183}
13182024-02-08T06:39:41.741Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.988838439Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183}
13192024-02-08T06:39:41.741Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.988858298Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183}
13202024-02-08T06:39:41.741Z{"msg":"Crucible fcfbb4b4-5782-4fdd-8e92-ca0e860ac80c has session id: 25ee67c8-6747-42e7-ac43-3f562bb8283b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.988891675Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13212024-02-08T06:39:41.742ZCrucible runtime is spawned
13222024-02-08T06:39:41.742Z{"msg":"Upstairs opts: Upstairs UUID: fcfbb4b4-5782-4fdd-8e92-ca0e860ac80c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.988951071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13232024-02-08T06:39:41.742Z{"msg":"Crucible stats registered with UUID: fcfbb4b4-5782-4fdd-8e92-ca0e860ac80c","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.988986888Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13242024-02-08T06:39:41.742Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989010777Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183}
13252024-02-08T06:39:41.742Z{"msg":"fcfbb4b4-5782-4fdd-8e92-ca0e860ac80c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989036915Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13262024-02-08T06:39:41.742Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989050634Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13272024-02-08T06:39:41.742Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.98908928Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13282024-02-08T06:39:41.742Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989122848Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13292024-02-08T06:39:41.742Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989149586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13302024-02-08T06:39:41.742Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989188793Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13312024-02-08T06:39:41.742Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989216101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13322024-02-08T06:39:41.742Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989277346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13332024-02-08T06:39:41.742Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989306494Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13342024-02-08T06:39:41.742Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989432345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13352024-02-08T06:39:41.743Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.98949348Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13362024-02-08T06:39:41.743Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989532687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13372024-02-08T06:39:41.743Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.989559075Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13382024-02-08T06:39:41.743Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.990073777Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13392024-02-08T06:39:41.743Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.990110574Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13402024-02-08T06:39:41.743Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.990128202Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13412024-02-08T06:39:41.743Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.990199847Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13422024-02-08T06:39:41.743Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.99069748Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13432024-02-08T06:39:41.743Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.990783373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13442024-02-08T06:39:41.743Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.991308804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13452024-02-08T06:39:41.743Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.991345631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13462024-02-08T06:39:41.743Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.991385728Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13472024-02-08T06:39:41.743Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.991424395Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13482024-02-08T06:39:41.743Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.991461393Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13492024-02-08T06:39:41.743Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.99149859Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13502024-02-08T06:39:41.743Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.991529337Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13512024-02-08T06:39:41.744Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.991545356Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13522024-02-08T06:39:41.744Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.991988273Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13532024-02-08T06:39:41.744Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992421141Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13542024-02-08T06:39:41.744Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992465217Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13552024-02-08T06:39:41.744Z{"msg":"[0]R flush_numbers[0..12]: [15, 19, 10, 1, 18, 15, 15, 18, 15, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992500595Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13562024-02-08T06:39:41.744Z{"msg":"[0]R generation[0..12]: [8, 11, 6, 1, 10, 8, 8, 10, 8, 11, 11, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992526833Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13572024-02-08T06:39:41.744Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.9925632Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13582024-02-08T06:39:41.744Z{"msg":"[1]R flush_numbers[0..12]: [15, 19, 10, 1, 18, 15, 15, 18, 15, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992598487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13592024-02-08T06:39:41.745Z{"msg":"[1]R generation[0..12]: [8, 11, 6, 1, 10, 8, 8, 10, 8, 11, 11, 10]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
13602024-02-08T06:39:41.745Z,"time":"2024-02-08T06:39:40.992635454Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13612024-02-08T06:39:41.745Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.99269033Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13622024-02-08T06:39:41.745Z{"msg":"[2]R flush_numbers[0..12]: [15, 19, 10, 1, 18, 15, 15, 18, 15, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992731457Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13632024-02-08T06:39:41.745Z{"msg":"[2]R generation[0..12]: [8, 11, 6, 1, 10, 8, 8, 10, 8, 11, 11, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992777114Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13642024-02-08T06:39:41.745Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992819891Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13652024-02-08T06:39:41.745Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992859708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13662024-02-08T06:39:41.745Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992881486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13672024-02-08T06:39:41.746Z{"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992896125Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13682024-02-08T06:39:41.746Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992909754Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13692024-02-08T06:39:41.746Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992943931Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13702024-02-08T06:39:41.746Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992977839Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13712024-02-08T06:39:41.746Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.992998487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13722024-02-08T06:39:41.750Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.993021235Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13732024-02-08T06:39:41.750Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.993059613Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13742024-02-08T06:39:41.751Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.99309543Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13752024-02-08T06:39:41.751Z{"msg":"fcfbb4b4-5782-4fdd-8e92-ca0e860ac80c is now active with session: 25ee67c8-6747-42e7-ac43-3f562bb8283b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.993126588Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13762024-02-08T06:39:41.751Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13772024-02-08T06:39:41.751Z,"time":"2024-02-08T06:39:40.993146296Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183,"session_id":"25ee67c8-6747-42e7-ac43-3f562bb8283b"}
13782024-02-08T06:39:41.751Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:40.993170714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183}
13792024-02-08T06:39:41.751ZLoading write count information from file "/tmp/test_repair_verify.data"
13802024-02-08T06:39:41.751ZRun Repair workload
13812024-02-08T06:39:41.751Z01/30 Write block 152 len 28672 data: 4 4 4 4 4 4 3
13822024-02-08T06:39:41.751Z02/30 Read block 505 len 36864
13832024-02-08T06:39:41.751Z03/30 Write block 86 len 40960 data: 3 3 2 2 2 3 4 4 4 4
13842024-02-08T06:39:41.752Z04/30 Read block 305 len 4096
13852024-02-08T06:39:41.752Z05/30 Read block 245 len 8192
13862024-02-08T06:39:41.752Z06/30 Write block 384 len 36864 data: 4 3 3 4 4 4 3 2 2
13872024-02-08T06:39:41.752Z07/30 Write block 357 len 8192 data: 4 2
13882024-02-08T06:39:41.752Z08/30 Write block 298 len 32768 data: 3 5 4 3 3 2 2 2
13892024-02-08T06:39:41.752Z09/30 Read block 359 len 36864
13902024-02-08T06:39:41.752Z10/30 Write block 509 len 8192 data: 3 3
13912024-02-08T06:39:41.752Z11/30 Read block 220 len 36864
13922024-02-08T06:39:41.752Z12/30 Read block 300 len 8192
13932024-02-08T06:39:41.752Z13/30 Read block 548 len 28672
13942024-02-08T06:39:41.756Z14/30 Read block 51 len 36864
13952024-02-08T06:39:41.756Z15/30 Write block 11 len 20480 data: 2 2 2 2 2
13962024-02-08T06:39:41.756Z16/30 Read block 555 len 36864
13972024-02-08T06:39:41.756Z17/30 Write block 247 len 28672 data: 2 2 2 2 2 2 2
13982024-02-08T06:39:41.756Z18/30 Write block 530 len 32768 data: 2 2 2 2 2 2 2 2
13992024-02-08T06:39:41.756Z19/30 Read block 579 len 24576
14002024-02-08T06:39:41.760Z20/30 Read block 307 len 8192
14012024-02-08T06:39:41.760Z21/30 Write block 316 len 40960 data: 2 2 2 2 2 2 2 2 2 2
14022024-02-08T06:39:41.760Z22/30 Read block 477 len 20480
14032024-02-08T06:39:41.763Z23/30 Read block 452 len 20480
14042024-02-08T06:39:41.763Z24/30 Read block 52 len 40960
14052024-02-08T06:39:41.764Z25/30 Write block 369 len 4096 data: 2
14062024-02-08T06:39:41.764Z26/30 Read block 459 len 36864
14072024-02-08T06:39:41.768Z27/30 Write block 470 len 24576 data: 2 2 3 3 3 3
14082024-02-08T06:39:41.768Z28/30 Write block 104 len 24576 data: 4 3 3 3 3 3
14092024-02-08T06:39:41.768Z29/30 Read block 22 len 20480
14102024-02-08T06:39:41.768Z30/30 Write block 17 len 12288 data: 2 2 2
14112024-02-08T06:39:41.768Z----------------------------------------------------------------
14122024-02-08T06:39:41.769Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14132024-02-08T06:39:41.769ZStates: Active Active Active
14142024-02-08T06:39:41.769ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14152024-02-08T06:39:41.769Z 1 Acked 1000 Write 7 Done Done Done false
14162024-02-08T06:39:41.769Z 2 Acked 1001 Read 9 Done Done Done false
14172024-02-08T06:39:41.769Z 3 Acked 1002 Write 10 Done Sent Done false
14182024-02-08T06:39:41.769Z 4 Acked 1003 Read 1 Done Sent Done false
14192024-02-08T06:39:41.769Z 5 Acked 1004 Read 2 Done Sent Done false
14202024-02-08T06:39:41.769Z 6 Acked 1005 Write 9 Done Sent Done false
14212024-02-08T06:39:41.770Z 7 Acked 1006 Write 2 Done Sent Done false
14222024-02-08T06:39:41.770Z 8 Acked 1007 Write 8 Done Sent Done false
14232024-02-08T06:39:41.770Z 9 Acked 1008 Read 9 Done Sent Done false
14242024-02-08T06:39:41.770Z 10 Acked 1009 Write 2 Done Sent Done false
14252024-02-08T06:39:41.770Z 11 Acked 1010 Read 9 Done Sent Done false
14262024-02-08T06:39:41.770Z 12 Acked 1011 Read 2 Done Sent Done false
14272024-02-08T06:39:41.770Z 13 Acked 1012 Read 7 Done Sent Done false
14282024-02-08T06:39:41.770Z 14 Acked 1013 Read 9 Done Sent Done false
14292024-02-08T06:39:41.770Z 15 Acked 1014 Write 5 Done Sent Done false
14302024-02-08T06:39:41.771Z 16 Acked 1015 Read 9 Done Sent Done false
14312024-02-08T06:39:41.771Z 17 Acked 1016 Write 7 Done Sent Done false
14322024-02-08T06:39:41.771Z 18 Acked 1017 Write 8 Done Sent Done false
14332024-02-08T06:39:41.771Z 19 Acked 1018 Read 6 Done Sent Done false
14342024-02-08T06:39:41.771Z 20 Acked 1019 Read 2 Done Sent Done false
14352024-02-08T06:39:41.771Z 21 Acked 1020 Write 10 Done Sent Done false
14362024-02-08T06:39:41.771Z 22 Acked 1021 Read 5 Done Sent Done false
14372024-02-08T06:39:41.771Z 23 Acked 1022 Read 5 Done Sent Done false
14382024-02-08T06:39:41.771Z 24 Acked 1023 Read 10 Done Sent Done false
14392024-02-08T06:39:41.772Z 25 Acked 1024 Write 1 Done Sent Done false
14402024-02-08T06:39:41.772Z 26 Acked 1025 Read 9 Done Sent Done false
14412024-02-08T06:39:41.772Z 27 Acked 1026 Write 6 Done Sent Done false
14422024-02-08T06:39:41.772Z 28 Acked 1027 Write 6 Done Sent Done false
14432024-02-08T06:39:41.772Z 29 Acked 1028 Read 5 Done Sent Done false
14442024-02-08T06:39:41.772Z 30 Acked 1029 Write 3 Sent Sent Sent false
14452024-02-08T06:39:41.772Z STATES DS:0 DS:1 DS:2 TOTAL
14462024-02-08T06:39:41.772Z New 0 0 0 0
14472024-02-08T06:39:41.773Z Sent 1 28 1 30
14482024-02-08T06:39:41.773Z Done 29 2 29 60
14492024-02-08T06:39:41.773Z Skipped 0 0 0 0
14502024-02-08T06:39:41.773Z Error 0 0 0 0
14512024-02-08T06:39:41.773ZLast Flush: 0 0 0
14522024-02-08T06:39:41.773ZDownstairs last five completed:
14532024-02-08T06:39:41.773ZUpstairs last five completed: 30 29 28 27 26
14542024-02-08T06:39:41.773Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:41.022090843Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1183}
14552024-02-08T06:39:41.777ZWrote out file "/tmp/test_repair_verify.data"
14562024-02-08T06:39:41.777Z
14572024-02-08T06:39:41.777ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14582024-02-08T06:39:41.794ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14592024-02-08T06:39:41.794Z 0 000-019 8 8 8 15 15 15 T F T <---
14602024-02-08T06:39:41.794Z 1 020-039 11 11 11 19 19 19 F F F
14612024-02-08T06:39:41.794Z 2 040-059 6 6 6 10 10 10 F F F
14622024-02-08T06:39:41.794Z 3 060-079 1 1 1 1 1 1 F F F
14632024-02-08T06:39:41.794Z 4 080-099 10 10 10 18 18 18 T F T <---
14642024-02-08T06:39:41.795Z 5 100-119 8 8 8 15 15 15 T F T <---
14652024-02-08T06:39:41.795Z 6 120-139 8 8 8 15 15 15 F F F
14662024-02-08T06:39:41.795Z 7 140-159 10 10 10 18 18 18 T T T
14672024-02-08T06:39:41.800Z 8 160-179 8 8 8 15 15 15 F F F
14682024-02-08T06:39:41.800Z 9 180-199 11 11 11 19 19 19 F F F
14692024-02-08T06:39:41.800Z 10 200-219 11 11 11 19 19 19 F F F
14702024-02-08T06:39:41.800Z 11 220-239 10 10 10 17 17 17 F F F
14712024-02-08T06:39:41.801Z 12 240-259 2 2 2 2 2 2 T F T <---
14722024-02-08T06:39:41.801Z 13 260-279 10 10 10 18 18 18 F F F
14732024-02-08T06:39:41.801Z 14 280-299 10 10 10 18 18 18 T F T <---
14742024-02-08T06:39:41.801Z 15 300-319 4 4 4 4 4 4 T F T <---
14752024-02-08T06:39:41.801Z 16 320-339 8 8 8 15 15 15 T F T <---
14762024-02-08T06:39:41.801Z 17 340-359 10 10 10 18 18 18 T F T <---
14772024-02-08T06:39:41.801Z 18 360-379 3 3 3 3 3 3 T F T <---
14782024-02-08T06:39:41.801Z 19 380-399 6 6 6 13 13 13 T F T <---
14792024-02-08T06:39:41.801Z 20 400-419 7 7 7 14 14 14 F F F
14802024-02-08T06:39:41.801Z 21 420-439 7 7 7 14 14 14 F F F
14812024-02-08T06:39:41.801Z 22 440-459 10 10 10 16 16 16 F F F
14822024-02-08T06:39:41.801Z 23 460-479 11 11 11 19 19 19 T F T <---
14832024-02-08T06:39:41.801Z 24 480-499 8 8 8 15 15 15 F F F
14842024-02-08T06:39:41.801Z 25 500-519 11 11 11 19 19 19 T F T <---
14852024-02-08T06:39:41.802Z 26 520-539 8 8 8 15 15 15 T F T <---
14862024-02-08T06:39:41.802Z 27 540-559 8 8 8 15 15 15 F F F
14872024-02-08T06:39:41.802Z 28 560-579 10 10 10 18 18 18 F F F
14882024-02-08T06:39:41.802Z 29 580-599 6 6 6 13 13 13 F F F
14892024-02-08T06:39:41.802ZMax gen: 11, Max flush: 19
14902024-02-08T06:39:41.802ZError: Difference in extent metadata found!
14912024-02-08T06:39:41.802ZA Difference in extent metadata is expected here
14922024-02-08T06:39:41.802ZOn loop 6
14932024-02-08T06:39:41.802Z
14942024-02-08T06:39:41.806ZVerifying data now
14952024-02-08T06:39:46.474ZNo change after verify
14962024-02-08T06:39:46.475ZLoop: 6 Downstairs dump after verify (and repair):
14972024-02-08T06:39:46.489ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14982024-02-08T06:39:46.489Z 0 000-019 12 12 12 20 20 20 F F F
14992024-02-08T06:39:46.490Z 1 020-039 11 11 11 19 19 19 F F F
15002024-02-08T06:39:46.490Z 2 040-059 6 6 6 10 10 10 F F F
15012024-02-08T06:39:46.490Z 3 060-079 1 1 1 1 1 1 F F F
15022024-02-08T06:39:46.490Z 4 080-099 12 12 12 20 20 20 F F F
15032024-02-08T06:39:46.491Z 5 100-119 12 12 12 20 20 20 F F F
15042024-02-08T06:39:46.491Z 6 120-139 8 8 8 15 15 15 F F F
15052024-02-08T06:39:46.491Z 7 140-159 12 12 12 20 20 20 F F F
15062024-02-08T06:39:46.496Z 8 160-179 8 8 8 15 15 15 F F F
15072024-02-08T06:39:46.496Z 9 180-199 11 11 11 19 19 19 F F F
15082024-02-08T06:39:46.496Z 10 200-219 11 11 11 19 19 19 F F F
15092024-02-08T06:39:46.497Z 11 220-239 10 10 10 17 17 17 F F F
15102024-02-08T06:39:46.497Z 12 240-259 12 12 12 20 20 20 F F F
15112024-02-08T06:39:46.497Z 13 260-279 10 10 10 18 18 18 F F F
15122024-02-08T06:39:46.497Z 14 280-299 12 12 12 20 20 20 F F F
15132024-02-08T06:39:46.497Z 15 300-319 12 12 12 20 20 20 F F F
15142024-02-08T06:39:46.498Z 16 320-339 12 12 12 20 20 20 F F F
15152024-02-08T06:39:46.498Z 17 340-359 12 12 12 20 20 20 F F F
15162024-02-08T06:39:46.498Z 18 360-379 12 12 12 20 20 20 F F F
15172024-02-08T06:39:46.499Z 19 380-399 12 12 12 20 20 20 F F F
15182024-02-08T06:39:46.499Z 20 400-419 7 7 7 14 14 14 F F F
15192024-02-08T06:39:46.499Z 21 420-439 7 7 7 14 14 14 F F F
15202024-02-08T06:39:46.499Z 22 440-459 10 10 10 16 16 16 F F F
15212024-02-08T06:39:46.500Z 23 460-479 12 12 12 20 20 20 F F F
15222024-02-08T06:39:46.500Z 24 480-499 8 8 8 15 15 15 F F F
15232024-02-08T06:39:46.500Z 25 500-519 12 12 12 20 20 20 F F F
15242024-02-08T06:39:46.500Z 26 520-539 12 12 12 20 20 20 F F F
15252024-02-08T06:39:46.501Z 27 540-559 8 8 8 15 15 15 F F F
15262024-02-08T06:39:46.501Z 28 560-579 10 10 10 18 18 18 F F F
15272024-02-08T06:39:46.501Z 29 580-599 6 6 6 13 13 13 F F F
15282024-02-08T06:39:46.501ZMax gen: 12, Max flush: 20
15292024-02-08T06:39:46.501Z
15302024-02-08T06:39:46.501ZBegin loop 7
15312024-02-08T06:39:46.502ZDownstairs to restart: 1
15322024-02-08T06:39:46.509Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.762681359Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191}
15332024-02-08T06:39:46.509Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
15342024-02-08T06:39:46.509Z,"time":"2024-02-08T06:39:45.762976977Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191}
15352024-02-08T06:39:46.509Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763014134Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191}
15362024-02-08T06:39:46.515Z{"msg":"Crucible 934918be-b9ae-41d3-bb44-9e8063334fa7 has session id: f7b778c8-48e8-4165-b491-b02f3de1fa18","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763053931Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15372024-02-08T06:39:46.515Z{"msg":"Upstairs opts: Upstairs UUID: 934918be-b9ae-41d3-bb44-9e8063334fa7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763103047Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15382024-02-08T06:39:46.516Z{"msg":"Crucible stats registered with UUID: 934918be-b9ae-41d3-bb44-9e8063334fa7","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763150494Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15392024-02-08T06:39:46.516Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763189311Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191}
15402024-02-08T06:39:46.516Z{"msg":"934918be-b9ae-41d3-bb44-9e8063334fa7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.76320513Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15412024-02-08T06:39:46.516Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763242547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15422024-02-08T06:39:46.516Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763273004Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15432024-02-08T06:39:46.516Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763304732Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15442024-02-08T06:39:46.517Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763325171Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15452024-02-08T06:39:46.517Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763352559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15462024-02-08T06:39:46.517Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763392496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15472024-02-08T06:39:46.517Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763426433Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15482024-02-08T06:39:46.517Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.76347486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15492024-02-08T06:39:46.517Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763509627Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15502024-02-08T06:39:46.517Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763540164Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15512024-02-08T06:39:46.518Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:45.763574322Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15522024-02-08T06:39:46.518Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:45.763611299Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15532024-02-08T06:39:46.518Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:45.763647896Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15542024-02-08T06:39:46.518Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763694453Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15552024-02-08T06:39:46.518Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763750119Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15562024-02-08T06:39:46.518Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763770997Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15572024-02-08T06:39:46.519Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.763799045Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15582024-02-08T06:39:46.519Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.764116351Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15592024-02-08T06:39:46.519Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.764156358Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15602024-02-08T06:39:46.519Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.764183556Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15612024-02-08T06:39:46.519Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.764204095Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15622024-02-08T06:39:46.520Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.764817509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15632024-02-08T06:39:46.520Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.764857166Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15642024-02-08T06:39:46.520Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.765411284Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15652024-02-08T06:39:46.520Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.765464Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15662024-02-08T06:39:46.520Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.765492698Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15672024-02-08T06:39:46.521Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.765517866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15682024-02-08T06:39:46.521Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.765533175Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15692024-02-08T06:39:46.521Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:45.765554583Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15702024-02-08T06:39:47.511Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.764945031Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15712024-02-08T06:39:47.516Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.765331372Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15722024-02-08T06:39:47.517Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.766172979Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15732024-02-08T06:39:47.517Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.766208506Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15742024-02-08T06:39:47.517Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.76668201Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15752024-02-08T06:39:47.517Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767240988Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15762024-02-08T06:39:47.517Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767291225Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15772024-02-08T06:39:47.517Z{"msg":"[0]R flush_numbers[0..12]: [20, 19, 10, 1, 20, 20, 15, 20, 15, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767320283Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15782024-02-08T06:39:47.517Z{"msg":"[0]R generation[0..12]: [12, 11, 6, 1, 12, 12, 8, 12, 8, 11, 11, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767339711Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15792024-02-08T06:39:47.517Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767367579Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15802024-02-08T06:39:47.517Z{"msg":"[1]R flush_numbers[0..12]: [20, 19, 10, 1, 20, 20, 15, 20, 15, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767394077Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15812024-02-08T06:39:47.517Z{"msg":"[1]R generation[0..12]: [12, 11, 6, 1, 12, 12, 8, 12, 8, 11, 11, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767412316Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15822024-02-08T06:39:47.517Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767437854Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
15832024-02-08T06:39:47.517Zdownstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15842024-02-08T06:39:47.517Z{"msg":"[2]R flush_numbers[0..12]: [20, 19, 10, 1, 20, 20, 15, 20, 15, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767473031Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15852024-02-08T06:39:47.518Z{"msg":"[2]R generation[0..12]: [12, 11, 6, 1, 12, 12, 8, 12, 8, 11, 11, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767497909Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15862024-02-08T06:39:47.518Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767516168Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15872024-02-08T06:39:47.518Z{"msg":"Max found gen is 13","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767541616Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15882024-02-08T06:39:47.518Z{"msg":"Generation requested: 14 >= found:13","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767559055Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15892024-02-08T06:39:47.518Z{"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767575123Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15902024-02-08T06:39:47.518Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767592132Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15912024-02-08T06:39:47.518Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.76761699Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15922024-02-08T06:39:47.518Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767632609Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15932024-02-08T06:39:47.518Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767657087Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15942024-02-08T06:39:47.518Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767673786Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15952024-02-08T06:39:47.518Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767698714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15962024-02-08T06:39:47.518Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767714243Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15972024-02-08T06:39:47.518Z{"msg":"934918be-b9ae-41d3-bb44-9e8063334fa7 is now active with session: f7b778c8-48e8-4165-b491-b02f3de1fa18","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767738621Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15982024-02-08T06:39:47.518Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.76775469Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191,"session_id":"f7b778c8-48e8-4165-b491-b02f3de1fa18"}
15992024-02-08T06:39:47.518Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:46.767770619Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191}
16002024-02-08T06:39:47.519ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
16012024-02-08T06:39:47.519ZLoading write count information from file "/tmp/test_repair_verify.data"
16022024-02-08T06:39:47.519ZRun Repair workload
16032024-02-08T06:39:47.519Z01/30 Write block 318 len 12288 data: 3 3 3
16042024-02-08T06:39:47.522Z02/30 Flush
16052024-02-08T06:39:47.522Z03/30 Write block 334 len 16384 data: 3 3 4 4
16062024-02-08T06:39:47.522Z04/30 Read block 296 len 16384
16072024-02-08T06:39:47.522Z05/30 Read block 251 len 4096
16082024-02-08T06:39:47.523Z06/30 Read block 74 len 36864
16092024-02-08T06:39:47.523Z07/30 Read block 393 len 8192
16102024-02-08T06:39:47.523Z08/30 Read block 11 len 8192
16112024-02-08T06:39:47.523Z09/30 Write block 56 len 8192 data: 2 2
16122024-02-08T06:39:47.523Z10/30 Read block 320 len 40960
16132024-02-08T06:39:47.526Z11/30 Read block 550 len 24576
16142024-02-08T06:39:47.526Z12/30 Read block 517 len 16384
16152024-02-08T06:39:47.526Z13/30 Read block 35 len 36864
16162024-02-08T06:39:47.529Z14/30 Read block 215 len 12288
16172024-02-08T06:39:47.529Z15/30 Read block 85 len 36864
16182024-02-08T06:39:47.529Z16/30 Write block 80 len 40960 data: 2 2 3 3 3 3 4 4 3 3
16192024-02-08T06:39:47.529Z17/30 Read block 29 len 16384
16202024-02-08T06:39:47.532Z18/30 Read block 288 len 32768
16212024-02-08T06:39:47.532Z19/30 Read block 274 len 16384
16222024-02-08T06:39:47.532Z20/30 Read block 96 len 24576
16232024-02-08T06:39:47.535Z21/30 Read block 429 len 40960
16242024-02-08T06:39:47.535Z22/30 Read block 377 len 40960
16252024-02-08T06:39:47.535Z23/30 Write block 83 len 16384 data: 4 4 4 5
16262024-02-08T06:39:47.536Z24/30 Read block 165 len 40960
16272024-02-08T06:39:47.541Z25/30 Read block 285 len 24576
16282024-02-08T06:39:47.541Z26/30 Read block 47 len 4096
16292024-02-08T06:39:47.541Z27/30 Write block 394 len 4096 data: 2
16302024-02-08T06:39:47.545Z28/30 Write block 504 len 28672 data: 3 3 3 3 3 4 4
16312024-02-08T06:39:47.545Z29/30 Write block 558 len 12288 data: 4 4 4
16322024-02-08T06:39:47.545Z30/30 Write block 313 len 32768 data: 2 2 2 3 3 4 4 4
16332024-02-08T06:39:47.545Z----------------------------------------------------------------
16342024-02-08T06:39:47.545Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:28
16352024-02-08T06:39:47.545ZStates: Active Active Active
16362024-02-08T06:39:47.545ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16372024-02-08T06:39:47.545Z 3 Acked 1002 Write 4 Done Done Done false
16382024-02-08T06:39:47.545Z 4 Acked 1003 Read 4 Done Done Done false
16392024-02-08T06:39:47.545Z 5 Acked 1004 Read 1 Done Done Done false
16402024-02-08T06:39:47.546Z 6 Acked 1005 Read 9 Done Sent Done false
16412024-02-08T06:39:47.546Z 7 Acked 1006 Read 2 Done Sent Done false
16422024-02-08T06:39:47.546Z 8 Acked 1007 Read 2 Done Sent Done false
16432024-02-08T06:39:47.546Z 9 Acked 1008 Write 2 Done Sent Done false
16442024-02-08T06:39:47.546Z 10 Acked 1009 Read 10 Done Sent Done false
16452024-02-08T06:39:47.546Z 11 Acked 1010 Read 6 Done Sent Done false
16462024-02-08T06:39:47.546Z 12 Acked 1011 Read 4 Done Sent Done false
16472024-02-08T06:39:47.546Z 13 Acked 1012 Read 9 Done Sent Done false
16482024-02-08T06:39:47.546Z 14 Acked 1013 Read 3 Done Sent Done false
16492024-02-08T06:39:47.546Z 15 Acked 1014 Read 9 Done Sent Done false
16502024-02-08T06:39:47.546Z 16 Acked 1015 Write 10 Done Sent Done false
16512024-02-08T06:39:47.546Z 17 Acked 1016 Read 4 Done Sent Done false
16522024-02-08T06:39:47.546Z 18 Acked 1017 Read 8 Done Sent Done false
16532024-02-08T06:39:47.546Z 19 Acked 1018 Read 4 Done Sent Done false
16542024-02-08T06:39:47.547Z 20 Acked 1019 Read 6 Done Sent Done false
16552024-02-08T06:39:47.547Z 21 Acked 1020 Read 10 Done Sent Done false
16562024-02-08T06:39:47.547Z 22 Acked 1021 Read 10 Done Sent Done false
16572024-02-08T06:39:47.547Z 23 Acked 1022 Write 4 Done Sent Done false
16582024-02-08T06:39:47.547Z 24 Acked 1023 Read 10 Done Sent Done false
16592024-02-08T06:39:47.547Z 25 Acked 1024 Read 6 Done Sent Done false
16602024-02-08T06:39:47.547Z 26 Acked 1025 Read 1 Done Sent Done false
16612024-02-08T06:39:47.547Z 27 Acked 1026 Write 1 Done Sent Done false
16622024-02-08T06:39:47.547Z 28 Acked 1027 Write 7 Done Sent Done false
16632024-02-08T06:39:47.547Z 29 Acked 1028 Write 3 Done Sent Done false
16642024-02-08T06:39:47.547Z 30 Acked 1029 Write 8 Sent Sent Sent false
16652024-02-08T06:39:47.547Z STATES DS:0 DS:1 DS:2 TOTAL
16662024-02-08T06:39:47.547Z New 0 0 0 0
16672024-02-08T06:39:47.547Z Sent 1 25 1 27
16682024-02-08T06:39:47.547Z Done 27 3 27 57
16692024-02-08T06:39:47.547Z Skipped 0 0 0 0
16702024-02-08T06:39:47.548Z Error 0 0 0 0
16712024-02-08T06:39:47.548ZLast Flush: 1001 1001 1001
16722024-02-08T06:39:47.548ZDownstairs last five completed: 1001 1000
16732024-02-08T06:39:47.548ZUpstairs last five completed: 30 29 28 27 26
16742024-02-08T06:39:47.548Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:46.795080429Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1191}
16752024-02-08T06:39:47.548ZWrote out file "/tmp/test_repair_verify.data"
16762024-02-08T06:39:47.551Z
16772024-02-08T06:39:47.551ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16782024-02-08T06:39:47.566ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16792024-02-08T06:39:47.566Z 0 000-019 12 12 12 20 20 20 F F F
16802024-02-08T06:39:47.566Z 1 020-039 11 11 11 19 19 19 F F F
16812024-02-08T06:39:47.567Z 2 040-059 6 6 6 10 10 10 T F T <---
16822024-02-08T06:39:47.567Z 3 060-079 1 1 1 1 1 1 F F F
16832024-02-08T06:39:47.567Z 4 080-099 12 12 12 20 20 20 T F T <---
16842024-02-08T06:39:47.570Z 5 100-119 12 12 12 20 20 20 F F F
16852024-02-08T06:39:47.570Z 6 120-139 8 8 8 15 15 15 F F F
16862024-02-08T06:39:47.570Z 7 140-159 12 12 12 20 20 20 F F F
16872024-02-08T06:39:47.570Z 8 160-179 8 8 8 15 15 15 F F F
16882024-02-08T06:39:47.570Z 9 180-199 11 11 11 19 19 19 F F F
16892024-02-08T06:39:47.570Z 10 200-219 11 11 11 19 19 19 F F F
16902024-02-08T06:39:47.570Z 11 220-239 10 10 10 17 17 17 F F F
16912024-02-08T06:39:47.571Z 12 240-259 12 12 12 20 20 20 F F F
16922024-02-08T06:39:47.571Z 13 260-279 10 10 10 18 18 18 F F F
16932024-02-08T06:39:47.571Z 14 280-299 12 12 12 20 20 20 F F F
16942024-02-08T06:39:47.571Z 15 300-319 14 14 14 21 21 21 T F T <---
16952024-02-08T06:39:47.571Z 16 320-339 14 14 14 21 21 21 T T T
16962024-02-08T06:39:47.571Z 17 340-359 12 12 12 20 20 20 F F F
16972024-02-08T06:39:47.571Z 18 360-379 12 12 12 20 20 20 F F F
16982024-02-08T06:39:47.571Z 19 380-399 12 12 12 20 20 20 T F T <---
16992024-02-08T06:39:47.571Z 20 400-419 7 7 7 14 14 14 F F F
17002024-02-08T06:39:47.571Z 21 420-439 7 7 7 14 14 14 F F F
17012024-02-08T06:39:47.571Z 22 440-459 10 10 10 16 16 16 F F F
17022024-02-08T06:39:47.571Z 23 460-479 12 12 12 20 20 20 F F F
17032024-02-08T06:39:47.571Z 24 480-499 8 8 8 15 15 15 F F F
17042024-02-08T06:39:47.571Z 25 500-519 12 12 12 20 20 20 T F T <---
17052024-02-08T06:39:47.571Z 26 520-539 12 12 12 20 20 20 F F F
17062024-02-08T06:39:47.572Z 27 540-559 8 8 8 15 15 15 T F T <---
17072024-02-08T06:39:47.572Z 28 560-579 10 10 10 18 18 18 T F T <---
17082024-02-08T06:39:47.572Z 29 580-599 6 6 6 13 13 13 F F F
17092024-02-08T06:39:47.572ZMax gen: 14, Max flush: 21
17102024-02-08T06:39:47.572ZError: Difference in extent metadata found!
17112024-02-08T06:39:47.572ZA Difference in extent metadata is expected here
17122024-02-08T06:39:47.572ZOn loop 7
17132024-02-08T06:39:47.572Z
17142024-02-08T06:39:47.575ZVerifying data now
17152024-02-08T06:39:52.005ZNo change after verify
17162024-02-08T06:39:52.005ZLoop: 7 Downstairs dump after verify (and repair):
17172024-02-08T06:39:52.020ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17182024-02-08T06:39:52.020Z 0 000-019 12 12 12 20 20 20 F F F
17192024-02-08T06:39:52.020Z 1 020-039 11 11 11 19 19 19 F F F
17202024-02-08T06:39:52.020Z 2 040-059 15 15 15 22 22 22 F F F
17212024-02-08T06:39:52.020Z 3 060-079 1 1 1 1 1 1 F F F
17222024-02-08T06:39:52.020Z 4 080-099 15 15 15 22 22 22 F F F
17232024-02-08T06:39:52.024Z 5 100-119 12 12 12 20 20 20 F F F
17242024-02-08T06:39:52.024Z 6 120-139 8 8 8 15 15 15 F F F
17252024-02-08T06:39:52.024Z 7 140-159 12 12 12 20 20 20 F F F
17262024-02-08T06:39:52.024Z 8 160-179 8 8 8 15 15 15 F F F
17272024-02-08T06:39:52.024Z 9 180-199 11 11 11 19 19 19 F F F
17282024-02-08T06:39:52.024Z 10 200-219 11 11 11 19 19 19 F F F
17292024-02-08T06:39:52.024Z 11 220-239 10 10 10 17 17 17 F F F
17302024-02-08T06:39:52.025Z 12 240-259 12 12 12 20 20 20 F F F
17312024-02-08T06:39:52.025Z 13 260-279 10 10 10 18 18 18 F F F
17322024-02-08T06:39:52.025Z 14 280-299 12 12 12 20 20 20 F F F
17332024-02-08T06:39:52.025Z 15 300-319 15 15 15 22 22 22 F F F
17342024-02-08T06:39:52.025Z 16 320-339 15 15 15 22 22 22 F F F
17352024-02-08T06:39:52.025Z 17 340-359 12 12 12 20 20 20 F F F
17362024-02-08T06:39:52.025Z 18 360-379 12 12 12 20 20 20 F F F
17372024-02-08T06:39:52.025Z 19 380-399 15 15 15 22 22 22 F F F
17382024-02-08T06:39:52.025Z 20 400-419 7 7 7 14 14 14 F F F
17392024-02-08T06:39:52.025Z 21 420-439 7 7 7 14 14 14 F F F
17402024-02-08T06:39:52.025Z 22 440-459 10 10 10 16 16 16 F F F
17412024-02-08T06:39:52.025Z 23 460-479 12 12 12 20 20 20 F F F
17422024-02-08T06:39:52.025Z 24 480-499 8 8 8 15 15 15 F F F
17432024-02-08T06:39:52.025Z 25 500-519 15 15 15 22 22 22 F F F
17442024-02-08T06:39:52.026Z 26 520-539 12 12 12 20 20 20 F F F
17452024-02-08T06:39:52.026Z 27 540-559 15 15 15 22 22 22 F F F
17462024-02-08T06:39:52.026Z 28 560-579 15 15 15 22 22 22 F F F
17472024-02-08T06:39:52.026Z 29 580-599 6 6 6 13 13 13 F F F
17482024-02-08T06:39:52.026ZMax gen: 15, Max flush: 22
17492024-02-08T06:39:52.026Z
17502024-02-08T06:39:52.026ZBegin loop 8
17512024-02-08T06:39:52.026ZDownstairs to restart: 1
17522024-02-08T06:39:52.042Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.295419363Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199}
17532024-02-08T06:39:52.042Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.295655996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199}
17542024-02-08T06:39:52.048Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.295685273Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":Crucible runtime is spawned
17552024-02-08T06:39:52.048Z1199}
17562024-02-08T06:39:52.048Z{"msg":"Crucible 0b0423ba-b556-4402-b1b7-5192d3b23944 has session id: 78878c85-0e44-4652-a947-19b365c29f09","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.29572864Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17572024-02-08T06:39:52.048Z{"msg":"Upstairs opts: Upstairs UUID: 0b0423ba-b556-4402-b1b7-5192d3b23944, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.295766857Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17582024-02-08T06:39:52.048Z{"msg":"Crucible stats registered with UUID: 0b0423ba-b556-4402-b1b7-5192d3b23944","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.295793135Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17592024-02-08T06:39:52.048Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.295824643Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199}
17602024-02-08T06:39:52.048Z{"msg":"0b0423ba-b556-4402-b1b7-5192d3b23944 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.295890248Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17612024-02-08T06:39:52.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.295940644Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17622024-02-08T06:39:52.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296028168Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17632024-02-08T06:39:52.049Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296070004Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17642024-02-08T06:39:52.049Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296103812Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17652024-02-08T06:39:52.049Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296155128Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17662024-02-08T06:39:52.049Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296189145Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17672024-02-08T06:39:52.049Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296229162Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17682024-02-08T06:39:52.049Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.29625636Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17692024-02-08T06:39:52.049Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296304857Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17702024-02-08T06:39:52.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296365942Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17712024-02-08T06:39:52.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296425008Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17722024-02-08T06:39:52.049Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:51.296478644Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17732024-02-08T06:39:52.049Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:51.296507921Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17742024-02-08T06:39:52.049Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:51.296551668Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17752024-02-08T06:39:52.049Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296566667Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17762024-02-08T06:39:52.049Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296580306Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17772024-02-08T06:39:52.049Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.296776701Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17782024-02-08T06:39:52.050Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.297113086Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17792024-02-08T06:39:52.050Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.297150343Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17802024-02-08T06:39:52.050Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.297175541Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17812024-02-08T06:39:52.050Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.297217168Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17822024-02-08T06:39:52.050Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.297744279Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17832024-02-08T06:39:52.050Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.297897967Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17842024-02-08T06:39:52.050Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.298323225Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17852024-02-08T06:39:52.050Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.298360592Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17862024-02-08T06:39:52.050Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.298376541Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17872024-02-08T06:39:52.050Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.298403049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17882024-02-08T06:39:52.050Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.298417138Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17892024-02-08T06:39:52.050Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:51.298431097Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17902024-02-08T06:39:53.045Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.298046408Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17912024-02-08T06:39:53.045Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.29815676Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17922024-02-08T06:39:53.052Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.299005576Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17932024-02-08T06:39:53.052Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.299039983Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17942024-02-08T06:39:53.053Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.299560964Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17952024-02-08T06:39:53.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300072896Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17962024-02-08T06:39:53.053Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300124512Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17972024-02-08T06:39:53.053Z{"msg":"[0]R flush_numbers[0..12]: [20, 19, 22, 1, 22, 20, 15, 20, 15, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.30014831Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17982024-02-08T06:39:53.054Z{"msg":"[0]R generation[0..12]: [12, 11, 15, 1, 15, 12, 8, 12, 8, 11, 11, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300167129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
17992024-02-08T06:39:53.054Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300185267Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18002024-02-08T06:39:53.054Z{"msg":"[1]R flush_numbers[0..12]: [20, 19, 22, 1, 22, 20, 15, 20, 15, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300213525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18012024-02-08T06:39:53.054Z{"msg":"[1]R generation[0..12]: [12, 11, 15, 1, 15, 12, 8, 12, 8, 11, 11, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300231354Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18022024-02-08T06:39:53.054Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300257082Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18032024-02-08T06:39:53.054Z{"msg":"[2]R flush_numbers[0..12]: [20, 19, 22, 1, 22, 20, 15, 20, 15, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300274841Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18042024-02-08T06:39:53.054Z{Wait for a query_work_queue command to finish before sending IO
18052024-02-08T06:39:53.055Z"msg":"[2]R generation[0..12]: [12, 11, 15, 1, 15, 12, 8, 12, 8, 11, 11, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300301578Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18062024-02-08T06:39:53.055Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300319687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18072024-02-08T06:39:53.055Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300337756Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18082024-02-08T06:39:53.055Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300355005Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18092024-02-08T06:39:53.055Z{"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300379293Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18102024-02-08T06:39:53.055Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.30041033Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18112024-02-08T06:39:53.055Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300424959Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18122024-02-08T06:39:53.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300438158Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18132024-02-08T06:39:53.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300451647Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18142024-02-08T06:39:53.056Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300471916Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18152024-02-08T06:39:53.056Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300485605Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18162024-02-08T06:39:53.056Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300509933Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18172024-02-08T06:39:53.056Z{"msg":"0b0423ba-b556-4402-b1b7-5192d3b23944 is now active with session: 78878c85-0e44-4652-a947-19b365c29f09","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300523372Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18182024-02-08T06:39:53.056Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.300535421Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199,"session_id":"78878c85-0e44-4652-a947-19b365c29f09"}
18192024-02-08T06:39:53.056Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:52.30054846Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199}
18202024-02-08T06:39:53.056ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
18212024-02-08T06:39:53.056ZLoading write count information from file "/tmp/test_repair_verify.data"
18222024-02-08T06:39:53.056ZRun Repair workload
18232024-02-08T06:39:53.057Z01/30 Write block 194 len 4096 data: 2
18242024-02-08T06:39:53.057Z02/30 Write block 391 len 20480 data: 3 3 2 3 3
18252024-02-08T06:39:53.061Z03/30 Write block 30 len 36864 data: 2 2 2 3 2 2 2 3 3
18262024-02-08T06:39:53.062Z04/30 Read block 521 len 40960
18272024-02-08T06:39:53.062Z05/30 Read block 128 len 36864
18282024-02-08T06:39:53.062Z06/30 Write block 389 len 4096 data: 5
18292024-02-08T06:39:53.062Z07/30 Read block 253 len 36864
18302024-02-08T06:39:53.062Z08/30 Write block 229 len 32768 data: 4 3 3 3 2 2 2 2
18312024-02-08T06:39:53.063Z09/30 Read block 569 len 24576
18322024-02-08T06:39:53.063Z10/30 Write block 158 len 32768 data: 4 3 3 3 4 4 5 4
18332024-02-08T06:39:53.063Z11/30 Flush
18342024-02-08T06:39:53.063Z12/30 Write block 249 len 16384 data: 3 3 3 3
18352024-02-08T06:39:53.063Z13/30 Read block 579 len 8192
18362024-02-08T06:39:53.068Z14/30 Read block 338 len 24576
18372024-02-08T06:39:53.068Z15/30 Write block 48 len 40960 data: 3 2 2 2 2 2 2 2 3 3
18382024-02-08T06:39:53.069Z16/30 Read block 180 len 32768
18392024-02-08T06:39:53.069Z17/30 Read block 56 len 40960
18402024-02-08T06:39:53.069Z18/30 Flush
18412024-02-08T06:39:53.073Z19/30 Flush
18422024-02-08T06:39:53.074Z20/30 Write block 207 len 8192 data: 2 2
18432024-02-08T06:39:53.074Z21/30 Write block 452 len 16384 data: 2 2 2 3
18442024-02-08T06:39:53.074Z22/30 Read block 91 len 4096
18452024-02-08T06:39:53.074Z23/30 Flush
18462024-02-08T06:39:53.074Z24/30 Write block 450 len 40960 data: 2 2 3 3 3 4 4 4 4 3
18472024-02-08T06:39:53.074Z25/30 Read block 181 len 28672
18482024-02-08T06:39:53.079Z26/30 Read block 1 len 12288
18492024-02-08T06:39:53.079Z27/30 Read block 114 len 28672
18502024-02-08T06:39:53.079Z28/30 Write block 90 len 32768 data: 3 4 5 5 5 5 3 3
18512024-02-08T06:39:53.080Z29/30 Read block 266 len 8192
18522024-02-08T06:39:53.080Z30/30 Read block 255 len 24576
18532024-02-08T06:39:53.084Z----------------------------------------------------------------
18542024-02-08T06:39:53.084Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18552024-02-08T06:39:53.084ZStates: Active Active Active
18562024-02-08T06:39:53.085ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18572024-02-08T06:39:53.085Z 1 Acked 1000 Write 1 Done Done Done false
18582024-02-08T06:39:53.085Z 2 Acked 1001 Write 5 Done Sent Done false
18592024-02-08T06:39:53.085Z 3 Acked 1002 Write 9 Done Sent Done false
18602024-02-08T06:39:53.085Z 4 Acked 1003 Read 10 Done Sent Done false
18612024-02-08T06:39:53.086Z 5 Acked 1004 Read 9 Done Sent Done false
18622024-02-08T06:39:53.086Z 6 Acked 1005 Write 1 Done Sent Done false
18632024-02-08T06:39:53.086Z 7 Acked 1006 Read 9 Done Sent Done false
18642024-02-08T06:39:53.086Z 8 Acked 1007 Write 8 Done Sent Done false
18652024-02-08T06:39:53.086Z 9 Acked 1008 Read 6 Done Sent Done false
18662024-02-08T06:39:53.087Z 10 Acked 1009 Write 8 Done Sent Done false
18672024-02-08T06:39:53.087Z 11 Acked 1010 Flush 0 Done Sent Done false
18682024-02-08T06:39:53.087Z 12 Acked 1011 Write 4 Done Sent Done false
18692024-02-08T06:39:53.087Z 13 Acked 1012 Read 2 Done Sent Done false
18702024-02-08T06:39:53.088Z 14 Acked 1013 Read 6 Done Sent Done false
18712024-02-08T06:39:53.088Z 15 Acked 1014 Write 10 Done Sent Done false
18722024-02-08T06:39:53.088Z 16 Acked 1015 Read 8 Done Sent Done false
18732024-02-08T06:39:53.088Z 17 Acked 1016 Read 10 Done Sent Done false
18742024-02-08T06:39:53.088Z 18 Acked 1017 Flush 0 Done Sent Done false
18752024-02-08T06:39:53.089Z 19 Acked 1018 Flush 0 Done Sent Done false
18762024-02-08T06:39:53.089Z 20 Acked 1019 Write 2 Done Sent Done false
18772024-02-08T06:39:53.089Z 21 Acked 1020 Write 4 Done Sent Done false
18782024-02-08T06:39:53.090Z 22 Acked 1021 Read 1 Done Sent Done false
18792024-02-08T06:39:53.090Z 23 Acked 1022 Flush 0 Done Sent Done false
18802024-02-08T06:39:53.090Z 24 Acked 1023 Write 10 Done Sent Done false
18812024-02-08T06:39:53.090Z 25 Acked 1024 Read 7 Done Sent Done false
18822024-02-08T06:39:53.091Z 26 Acked 1025 Read 3 Done Sent Done false
18832024-02-08T06:39:53.091Z 27 Acked 1026 Read 7 Done Sent Done false
18842024-02-08T06:39:53.091Z 28 Acked 1027 Write 8 Done Sent Done false
18852024-02-08T06:39:53.091Z 29 Acked 1028 Read 2 Done Sent Done false
18862024-02-08T06:39:53.092Z 30 Acked 1029 Read 6 Done Sent Done false
18872024-02-08T06:39:53.092Z STATES DS:0 DS:1 DS:2 TOTAL
18882024-02-08T06:39:53.092Z New 0 0 0 0
18892024-02-08T06:39:53.092Z Sent 0 29 0 29
18902024-02-08T06:39:53.093Z Done 30 1 30 61
18912024-02-08T06:39:53.093Z Skipped 0 0 0 0
18922024-02-08T06:39:53.093Z Error 0 0 0 0
18932024-02-08T06:39:53.093ZLast Flush: 1022 0 1022
18942024-02-08T06:39:53.094ZDownstairs last five completed:
18952024-02-08T06:39:53.094ZUpstairs last five completed: 30 29 28 27 26
18962024-02-08T06:39:53.094Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:52.332446185Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1199}
18972024-02-08T06:39:53.094ZWrote out file "/tmp/test_repair_verify.data"
18982024-02-08T06:39:53.094Z
18992024-02-08T06:39:53.113ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
19002024-02-08T06:39:53.137ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19012024-02-08T06:39:53.137Z 0 000-019 12 12 12 20 20 20 F F F
19022024-02-08T06:39:53.137Z 1 020-039 16 11 16 23 19 23 F F F <---
19032024-02-08T06:39:53.137Z 2 040-059 16 15 16 24 22 24 F F F <---
19042024-02-08T06:39:53.138Z 3 060-079 1 1 1 1 1 1 F F F
19052024-02-08T06:39:53.138Z 4 080-099 15 15 15 22 22 22 T F T <---
19062024-02-08T06:39:53.138Z 5 100-119 12 12 12 20 20 20 F F F
19072024-02-08T06:39:53.139Z 6 120-139 8 8 8 15 15 15 F F F
19082024-02-08T06:39:53.139Z 7 140-159 16 12 16 23 20 23 F F F <---
19092024-02-08T06:39:53.139Z 8 160-179 16 8 16 23 15 23 F F F <---
19102024-02-08T06:39:53.140Z 9 180-199 16 11 16 23 19 23 F T F <---
19112024-02-08T06:39:53.140Z 10 200-219 16 11 16 26 19 26 F F F <---
19122024-02-08T06:39:53.140Z 11 220-239 16 10 16 23 17 23 F F F <---
19132024-02-08T06:39:53.140Z 12 240-259 16 12 16 24 20 24 F F F <---
19142024-02-08T06:39:53.140Z 13 260-279 10 10 10 18 18 18 F F F
19152024-02-08T06:39:53.140Z 14 280-299 12 12 12 20 20 20 F F F
19162024-02-08T06:39:53.140Z 15 300-319 15 15 15 22 22 22 F F F
19172024-02-08T06:39:53.140Z 16 320-339 15 15 15 22 22 22 F F F
19182024-02-08T06:39:53.141Z 17 340-359 12 12 12 20 20 20 F F F
19192024-02-08T06:39:53.141Z 18 360-379 12 12 12 20 20 20 F F F
19202024-02-08T06:39:53.141Z 19 380-399 16 15 16 23 22 23 F F F <---
19212024-02-08T06:39:53.141Z 20 400-419 7 7 7 14 14 14 F F F
19222024-02-08T06:39:53.141Z 21 420-439 7 7 7 14 14 14 F F F
19232024-02-08T06:39:53.141Z 22 440-459 16 10 16 26 16 26 T F T <---
19242024-02-08T06:39:53.142Z 23 460-479 12 12 12 20 20 20 F F F
19252024-02-08T06:39:53.142Z 24 480-499 8 8 8 15 15 15 F F F
19262024-02-08T06:39:53.142Z 25 500-519 15 15 15 22 22 22 F F F
19272024-02-08T06:39:53.142Z 26 520-539 12 12 12 20 20 20 F F F
19282024-02-08T06:39:53.142Z 27 540-559 15 15 15 22 22 22 F F F
19292024-02-08T06:39:53.142Z 28 560-579 15 15 15 22 22 22 F F F
19302024-02-08T06:39:53.143Z 29 580-599 6 6 6 13 13 13 F F F
19312024-02-08T06:39:53.143ZMax gen: 16, Max flush: 26
19322024-02-08T06:39:53.143ZError: Difference in extent metadata found!
19332024-02-08T06:39:53.143ZA Difference in extent metadata is expected here
19342024-02-08T06:39:53.143ZOn loop 8
19352024-02-08T06:39:53.144Z
19362024-02-08T06:39:53.144ZVerifying data now
19372024-02-08T06:39:57.653ZNo change after verify
19382024-02-08T06:39:57.653ZLoop: 8 Downstairs dump after verify (and repair):
19392024-02-08T06:39:57.668ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19402024-02-08T06:39:57.668Z 0 000-019 12 12 12 20 20 20 F F F
19412024-02-08T06:39:57.668Z 1 020-039 16 16 16 23 23 23 F F F
19422024-02-08T06:39:57.668Z 2 040-059 16 16 16 24 24 24 F F F
19432024-02-08T06:39:57.668Z 3 060-079 1 1 1 1 1 1 F F F
19442024-02-08T06:39:57.668Z 4 080-099 17 17 17 27 27 27 F F F
19452024-02-08T06:39:57.668Z 5 100-119 12 12 12 20 20 20 F F F
19462024-02-08T06:39:57.669Z 6 120-139 8 8 8 15 15 15 F F F
19472024-02-08T06:39:57.669Z 7 140-159 16 16 16 23 23 23 F F F
19482024-02-08T06:39:57.672Z 8 160-179 16 16 16 23 23 23 F F F
19492024-02-08T06:39:57.672Z 9 180-199 16 16 16 23 23 23 F F F
19502024-02-08T06:39:57.672Z 10 200-219 16 16 16 26 26 26 F F F
19512024-02-08T06:39:57.672Z 11 220-239 16 16 16 23 23 23 F F F
19522024-02-08T06:39:57.672Z 12 240-259 16 16 16 24 24 24 F F F
19532024-02-08T06:39:57.672Z 13 260-279 10 10 10 18 18 18 F F F
19542024-02-08T06:39:57.673Z 14 280-299 12 12 12 20 20 20 F F F
19552024-02-08T06:39:57.673Z 15 300-319 15 15 15 22 22 22 F F F
19562024-02-08T06:39:57.673Z 16 320-339 15 15 15 22 22 22 F F F
19572024-02-08T06:39:57.673Z 17 340-359 12 12 12 20 20 20 F F F
19582024-02-08T06:39:57.673Z 18 360-379 12 12 12 20 20 20 F F F
19592024-02-08T06:39:57.673Z 19 380-399 16 16 16 23 23 23 F F F
19602024-02-08T06:39:57.673Z 20 400-419 7 7 7 14 14 14 F F F
19612024-02-08T06:39:57.673Z 21 420-439 7 7 7 14 14 14 F F F
19622024-02-08T06:39:57.673Z 22 440-459 17 17 17 27 27 27 F F F
19632024-02-08T06:39:57.673Z 23 460-479 12 12 12 20 20 20 F F F
19642024-02-08T06:39:57.673Z 24 480-499 8 8 8 15 15 15 F F F
19652024-02-08T06:39:57.673Z 25 500-519 15 15 15 22 22 22 F F F
19662024-02-08T06:39:57.673Z 26 520-539 12 12 12 20 20 20 F F F
19672024-02-08T06:39:57.673Z 27 540-559 15 15 15 22 22 22 F F F
19682024-02-08T06:39:57.673Z 28 560-579 15 15 15 22 22 22 F F F
19692024-02-08T06:39:57.674Z 29 580-599 6 6 6 13 13 13 F F F
19702024-02-08T06:39:57.674ZMax gen: 17, Max flush: 27
19712024-02-08T06:39:57.674Z
19722024-02-08T06:39:57.674ZBegin loop 9
19732024-02-08T06:39:57.674ZDownstairs to restart: 0
19742024-02-08T06:39:57.688Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.941518165Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207}
19752024-02-08T06:39:57.693Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
19762024-02-08T06:39:57.694Z,"time":"2024-02-08T06:39:56.941804434Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207}
19772024-02-08T06:39:57.694Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.941861849Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207}
19782024-02-08T06:39:57.694Z{"msg":"Crucible 601be197-7a48-4702-bbaa-1aaf8971c3a1 has session id: 246c26d3-4018-4cda-9813-b14ec4098888","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.941902306Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19792024-02-08T06:39:57.694Z{"msg":"Upstairs opts: Upstairs UUID: 601be197-7a48-4702-bbaa-1aaf8971c3a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.941940873Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19802024-02-08T06:39:57.694Z{"msg":"Crucible stats registered with UUID: 601be197-7a48-4702-bbaa-1aaf8971c3a1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.941974151Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19812024-02-08T06:39:57.694Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942001419Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207}
19822024-02-08T06:39:57.694Z{"msg":"601be197-7a48-4702-bbaa-1aaf8971c3a1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942038366Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19832024-02-08T06:39:57.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942066474Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19842024-02-08T06:39:57.694Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942110631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19852024-02-08T06:39:57.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942151488Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19862024-02-08T06:39:57.694Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942176296Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19872024-02-08T06:39:57.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942209183Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19882024-02-08T06:39:57.694Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942268399Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19892024-02-08T06:39:57.695Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942298577Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19902024-02-08T06:39:57.695Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942325675Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19912024-02-08T06:39:57.695Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942350293Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19922024-02-08T06:39:57.695Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.94238465Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19932024-02-08T06:39:57.695Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942432096Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19942024-02-08T06:39:57.695Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.942464684Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19952024-02-08T06:39:57.695Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.943003464Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19962024-02-08T06:39:57.695Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.9430553Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19972024-02-08T06:39:57.695Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.943100786Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19982024-02-08T06:39:57.695Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.943138154Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
19992024-02-08T06:39:57.695Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.943661054Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20002024-02-08T06:39:57.695Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.94371158Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20012024-02-08T06:39:57.695Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.944194064Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20022024-02-08T06:39:57.695Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.944235561Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20032024-02-08T06:39:57.695Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.944365501Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20042024-02-08T06:39:57.695Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.944445805Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20052024-02-08T06:39:57.696Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.944481113Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20062024-02-08T06:39:57.696Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.944496461Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20072024-02-08T06:39:57.696Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.944511251Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20082024-02-08T06:39:57.696Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.944533109Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20092024-02-08T06:39:57.696Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.944860784Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20102024-02-08T06:39:57.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945365856Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20112024-02-08T06:39:57.703Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945397214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20122024-02-08T06:39:57.703Z{"msg":"[0]R flush_numbers[0..12]: [20, 23, 24, 1, 27, 20, 15, 23, 23, 23, 26, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945413283Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20132024-02-08T06:39:57.703Z{"msg":"[0]R generation[0..12]: [12, 16, 16, 1, 17, 12, 8, 16, 16, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945436301Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20142024-02-08T06:39:57.703Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945462749Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20152024-02-08T06:39:57.703Z{"msg":"[1]R flush_numbers[0..12]: [20, 23, 24, 1, 27, 20, 15, 23, 23, 23, 26, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945485347Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20162024-02-08T06:39:57.703Z{"msg":"[1]R generation[0..12]: [12, 16, 16, 1, 17, 12, 8, 16, 16, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945499766Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20172024-02-08T06:39:57.704Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945521255Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20182024-02-08T06:39:57.704Z{"msg":"[2]R flush_numbers[0..12]: [20, 23, 24, 1, 27, 20, 15, 23, 23, 23, 26, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945539933Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20192024-02-08T06:39:57.704ZWait for a query_work_queue command to finish before sending IO
20202024-02-08T06:39:57.704Z{"msg":"[2]R generation[0..12]: [12, 16, 16, 1, 17, 12, 8, 16, 16, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.94557892Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20212024-02-08T06:39:57.704Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945615908Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20222024-02-08T06:39:57.704Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945632416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20232024-02-08T06:39:57.704Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945659754Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20242024-02-08T06:39:57.704Z{"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945679553Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20252024-02-08T06:39:57.704Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.94572043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20262024-02-08T06:39:57.704Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945742778Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20272024-02-08T06:39:57.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945781545Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20282024-02-08T06:39:57.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945817002Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20292024-02-08T06:39:57.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945832591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20302024-02-08T06:39:57.705Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945860959Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20312024-02-08T06:39:57.705Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945874758Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20322024-02-08T06:39:57.705Z{"msg":"601be197-7a48-4702-bbaa-1aaf8971c3a1 is now active with session: 246c26d3-4018-4cda-9813-b14ec4098888","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945907806Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20332024-02-08T06:39:57.705Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945937433Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207,"session_id":"246c26d3-4018-4cda-9813-b14ec4098888"}
20342024-02-08T06:39:57.705Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:39:56.945952192Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207}
20352024-02-08T06:39:57.705ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20362024-02-08T06:39:57.705ZLoading write count information from file "/tmp/test_repair_verify.data"
20372024-02-08T06:39:57.705ZRun Repair workload
20382024-02-08T06:39:57.705Z01/30 Write block 222 len 36864 data: 3 3 4 4 4 4 5 5 4
20392024-02-08T06:39:57.705Z02/30 Write block 7 len 16384 data: 3 3 3 3
20402024-02-08T06:39:57.705Z03/30 Read block 314 len 36864
20412024-02-08T06:39:57.705Z04/30 Write block 517 len 16384 data: 2 3 3 3
20422024-02-08T06:39:57.705Z05/30 Write block 494 len 36864 data: 2 2 2 2 2 2 2 2 2
20432024-02-08T06:39:57.705Z06/30 Read block 333 len 40960
20442024-02-08T06:39:57.712Z07/30 Write block 226 len 20480 data: 5 5 6 6 5
20452024-02-08T06:39:57.712Z08/30 Read block 205 len 8192
20462024-02-08T06:39:57.712Z09/30 Write block 233 len 28672 data: 3 3 3 3 2 2 2
20472024-02-08T06:39:57.712Z10/30 Write block 547 len 32768 data: 3 3 3 2 2 2 3 3
20482024-02-08T06:39:57.712Z11/30 Write block 54 len 28672 data: 3 3 4 4 2 2 2
20492024-02-08T06:39:57.712Z12/30 Flush
20502024-02-08T06:39:57.712Z13/30 Write block 579 len 8192 data: 2 2
20512024-02-08T06:39:57.712Z14/30 Read block 542 len 36864
20522024-02-08T06:39:57.719Z15/30 Read block 7 len 20480
20532024-02-08T06:39:57.719Z16/30 Write block 583 len 8192 data: 2 2
20542024-02-08T06:39:57.719Z17/30 Read block 81 len 16384
20552024-02-08T06:39:57.719Z18/30 Write block 209 len 4096 data: 2
20562024-02-08T06:39:57.719Z19/30 Write block 546 len 16384 data: 3 4 4 4
20572024-02-08T06:39:57.719Z20/30 Read block 449 len 36864
20582024-02-08T06:39:57.719Z21/30 Write block 290 len 8192 data: 5 5
20592024-02-08T06:39:57.719Z22/30 Read block 580 len 4096
20602024-02-08T06:39:57.719Z23/30 Read block 501 len 36864
20612024-02-08T06:39:57.720Z24/30 Read block 241 len 4096
20622024-02-08T06:39:57.720Z25/30 Write block 184 len 24576 data: 2 2 2 2 2 2
20632024-02-08T06:39:57.720Z26/30 Write block 272 len 8192 data: 2 2
20642024-02-08T06:39:57.720Z27/30 Read block 196 len 4096
20652024-02-08T06:39:57.720Z28/30 Write block 483 len 28672 data: 3 3 3 3 3 3 3
20662024-02-08T06:39:57.723Z29/30 Write block 212 len 16384 data: 3 3 3 2
20672024-02-08T06:39:57.724Z30/30 Read block 463 len 8192
20682024-02-08T06:39:57.724Z----------------------------------------------------------------
20692024-02-08T06:39:57.724Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20702024-02-08T06:39:57.724ZStates: Active Active Active
20712024-02-08T06:39:57.724ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20722024-02-08T06:39:57.724Z 1 Acked 1000 Write 9 Done Done Done false
20732024-02-08T06:39:57.724Z 2 Acked 1001 Write 4 Done Done Done false
20742024-02-08T06:39:57.724Z 3 Acked 1002 Read 9 Done Done Done false
20752024-02-08T06:39:57.724Z 4 Acked 1003 Write 4 Done Done Done false
20762024-02-08T06:39:57.724Z 5 Acked 1004 Write 9 Done Done Done false
20772024-02-08T06:39:57.724Z 6 Acked 1005 Read 10 Sent Done Done false
20782024-02-08T06:39:57.724Z 7 Acked 1006 Write 5 Sent Done Done false
20792024-02-08T06:39:57.724Z 8 Acked 1007 Read 2 Sent Done Done false
20802024-02-08T06:39:57.724Z 9 Acked 1008 Write 7 Sent Done Done false
20812024-02-08T06:39:57.725Z 10 Acked 1009 Write 8 Sent Done Done false
20822024-02-08T06:39:57.725Z 11 Acked 1010 Write 7 Sent Done Done false
20832024-02-08T06:39:57.725Z 12 Acked 1011 Flush 0 Sent Done Done false
20842024-02-08T06:39:57.725Z 13 Acked 1012 Write 2 Sent Done Done false
20852024-02-08T06:39:57.725Z 14 Acked 1013 Read 9 Sent Done Done false
20862024-02-08T06:39:57.725Z 15 Acked 1014 Read 5 Sent Done Done false
20872024-02-08T06:39:57.725Z 16 Acked 1015 Write 2 Sent Done Done false
20882024-02-08T06:39:57.725Z 17 Acked 1016 Read 4 Sent Done Done false
20892024-02-08T06:39:57.725Z 18 Acked 1017 Write 1 Sent Done Done false
20902024-02-08T06:39:57.725Z 19 Acked 1018 Write 4 Sent Done Done false
20912024-02-08T06:39:57.725Z 20 Acked 1019 Read 9 Sent Done Done false
20922024-02-08T06:39:57.725Z 21 Acked 1020 Write 2 Sent Done Done false
20932024-02-08T06:39:57.725Z 22 Acked 1021 Read 1 Sent Done Done false
20942024-02-08T06:39:57.725Z 23 Acked 1022 Read 9 Sent Done Done false
20952024-02-08T06:39:57.725Z 24 Acked 1023 Read 1 Sent Done Done false
20962024-02-08T06:39:57.725Z 25 Acked 1024 Write 6 Sent Done Done false
20972024-02-08T06:39:57.726Z 26 Acked 1025 Write 2 Sent Done Done false
20982024-02-08T06:39:57.726Z 27 Acked 1026 Read 1 Sent Done Done false
20992024-02-08T06:39:57.726Z 28 Acked 1027 Write 7 Sent Done Done false
21002024-02-08T06:39:57.726Z 29 Acked 1028 Write 4 Sent Done Done false
21012024-02-08T06:39:57.726Z 30 Acked 1029 Read 2 Sent Done Done false
21022024-02-08T06:39:57.726Z STATES DS:0 DS:1 DS:2 TOTAL
21032024-02-08T06:39:57.726Z New 0 0 0 0
21042024-02-08T06:39:57.726Z Sent 25 0 0 25
21052024-02-08T06:39:57.726Z Done 5 30 30 65
21062024-02-08T06:39:57.726Z Skipped 0 0 0 0
21072024-02-08T06:39:57.726Z Error 0 0 0 0
21082024-02-08T06:39:57.726ZLast Flush: 0 1011 1011
21092024-02-08T06:39:57.726ZDownstairs last five completed:
21102024-02-08T06:39:57.726ZUpstairs last five completed: 30 29 28 27 26
21112024-02-08T06:39:57.726Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:39:56.973940241Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1207}
21122024-02-08T06:39:57.727ZWrote out file "/tmp/test_repair_verify.data"
21132024-02-08T06:39:57.730Z
21142024-02-08T06:39:57.730ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
21152024-02-08T06:39:57.746ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21162024-02-08T06:39:57.746Z 0 000-019 12 18 18 20 28 28 T F F <---
21172024-02-08T06:39:57.746Z 1 020-039 16 16 16 23 23 23 F F F
21182024-02-08T06:39:57.746Z 2 040-059 16 18 18 24 28 28 F F F <---
21192024-02-08T06:39:57.746Z 3 060-079 1 18 18 1 28 28 F F F <---
21202024-02-08T06:39:57.746Z 4 080-099 17 17 17 27 27 27 F F F
21212024-02-08T06:39:57.746Z 5 100-119 12 12 12 20 20 20 F F F
21222024-02-08T06:39:57.746Z 6 120-139 8 8 8 15 15 15 F F F
21232024-02-08T06:39:57.746Z 7 140-159 16 16 16 23 23 23 F F F
21242024-02-08T06:39:57.746Z 8 160-179 16 16 16 23 23 23 F F F
21252024-02-08T06:39:57.747Z 9 180-199 16 16 16 23 23 23 F T T <---
21262024-02-08T06:39:57.747Z 10 200-219 16 16 16 26 26 26 F T T <---
21272024-02-08T06:39:57.750Z 11 220-239 16 18 18 23 28 28 T F F <---
21282024-02-08T06:39:57.750Z 12 240-259 16 16 16 24 24 24 F F F
21292024-02-08T06:39:57.750Z 13 260-279 10 10 10 18 18 18 F T T <---
21302024-02-08T06:39:57.750Z 14 280-299 12 12 12 20 20 20 F T T <---
21312024-02-08T06:39:57.750Z 15 300-319 15 15 15 22 22 22 F F F
21322024-02-08T06:39:57.750Z 16 320-339 15 15 15 22 22 22 F F F
21332024-02-08T06:39:57.750Z 17 340-359 12 12 12 20 20 20 F F F
21342024-02-08T06:39:57.750Z 18 360-379 12 12 12 20 20 20 F F F
21352024-02-08T06:39:57.750Z 19 380-399 16 16 16 23 23 23 F F F
21362024-02-08T06:39:57.751Z 20 400-419 7 7 7 14 14 14 F F F
21372024-02-08T06:39:57.751Z 21 420-439 7 7 7 14 14 14 F F F
21382024-02-08T06:39:57.751Z 22 440-459 17 17 17 27 27 27 F F F
21392024-02-08T06:39:57.751Z 23 460-479 12 12 12 20 20 20 F F F
21402024-02-08T06:39:57.751Z 24 480-499 8 18 18 15 28 28 T T T <---
21412024-02-08T06:39:57.751Z 25 500-519 15 18 18 22 28 28 T F F <---
21422024-02-08T06:39:57.751Z 26 520-539 12 18 18 20 28 28 T F F <---
21432024-02-08T06:39:57.751Z 27 540-559 15 18 18 22 28 28 F T T <---
21442024-02-08T06:39:57.751Z 28 560-579 15 15 15 22 22 22 F T T <---
21452024-02-08T06:39:57.751Z 29 580-599 6 6 6 13 13 13 F T T <---
21462024-02-08T06:39:57.751ZMax gen: 18, Max flush: 28
21472024-02-08T06:39:57.751ZError: Difference in extent metadata found!
21482024-02-08T06:39:57.751ZA Difference in extent metadata is expected here
21492024-02-08T06:39:57.751ZOn loop 9
21502024-02-08T06:39:57.751Z
21512024-02-08T06:39:57.755ZVerifying data now
21522024-02-08T06:40:03.429ZNo change after verify
21532024-02-08T06:40:03.429ZLoop: 9 Downstairs dump after verify (and repair):
21542024-02-08T06:40:03.443ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21552024-02-08T06:40:03.443Z 0 000-019 18 18 18 28 28 28 F F F
21562024-02-08T06:40:03.443Z 1 020-039 16 16 16 23 23 23 F F F
21572024-02-08T06:40:03.443Z 2 040-059 18 18 18 28 28 28 F F F
21582024-02-08T06:40:03.443Z 3 060-079 18 18 18 28 28 28 F F F
21592024-02-08T06:40:03.444Z 4 080-099 17 17 17 27 27 27 F F F
21602024-02-08T06:40:03.444Z 5 100-119 12 12 12 20 20 20 F F F
21612024-02-08T06:40:03.444Z 6 120-139 8 8 8 15 15 15 F F F
21622024-02-08T06:40:03.444Z 7 140-159 16 16 16 23 23 23 F F F
21632024-02-08T06:40:03.451Z 8 160-179 16 16 16 23 23 23 F F F
21642024-02-08T06:40:03.451Z 9 180-199 19 19 19 29 29 29 F F F
21652024-02-08T06:40:03.451Z 10 200-219 19 19 19 29 29 29 F F F
21662024-02-08T06:40:03.451Z 11 220-239 18 18 18 28 28 28 F F F
21672024-02-08T06:40:03.451Z 12 240-259 16 16 16 24 24 24 F F F
21682024-02-08T06:40:03.451Z 13 260-279 19 19 19 29 29 29 F F F
21692024-02-08T06:40:03.451Z 14 280-299 19 19 19 29 29 29 F F F
21702024-02-08T06:40:03.451Z 15 300-319 15 15 15 22 22 22 F F F
21712024-02-08T06:40:03.451Z 16 320-339 15 15 15 22 22 22 F F F
21722024-02-08T06:40:03.451Z 17 340-359 12 12 12 20 20 20 F F F
21732024-02-08T06:40:03.451Z 18 360-379 12 12 12 20 20 20 F F F
21742024-02-08T06:40:03.451Z 19 380-399 16 16 16 23 23 23 F F F
21752024-02-08T06:40:03.451Z 20 400-419 7 7 7 14 14 14 F F F
21762024-02-08T06:40:03.451Z 21 420-439 7 7 7 14 14 14 F F F
21772024-02-08T06:40:03.452Z 22 440-459 17 17 17 27 27 27 F F F
21782024-02-08T06:40:03.452Z 23 460-479 12 12 12 20 20 20 F F F
21792024-02-08T06:40:03.452Z 24 480-499 19 19 19 29 29 29 F F F
21802024-02-08T06:40:03.452Z 25 500-519 18 18 18 28 28 28 F F F
21812024-02-08T06:40:03.452Z 26 520-539 18 18 18 28 28 28 F F F
21822024-02-08T06:40:03.452Z 27 540-559 19 19 19 29 29 29 F F F
21832024-02-08T06:40:03.452Z 28 560-579 19 19 19 29 29 29 F F F
21842024-02-08T06:40:03.452Z 29 580-599 19 19 19 29 29 29 F F F
21852024-02-08T06:40:03.452ZMax gen: 19, Max flush: 29
21862024-02-08T06:40:03.452Z
21872024-02-08T06:40:03.452ZBegin loop 10
21882024-02-08T06:40:03.452ZDownstairs to restart: 2
21892024-02-08T06:40:03.466Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.71625396Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215}
21902024-02-08T06:40:03.467Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716497022Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215}
21912024-02-08T06:40:03.467Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716516601Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215}
21922024-02-08T06:40:03.918Z{"msg":"Crucible 3a0d6ae6-1b25-4e4d-833e-89b3b2a9b403 has session id: d79271a6-d45e-4be5-aee7-be03b30ab2eb","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716547458Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
21932024-02-08T06:40:03.918ZCrucible runtime is spawned
21942024-02-08T06:40:03.918Z{"msg":"Upstairs opts: Upstairs UUID: 3a0d6ae6-1b25-4e4d-833e-89b3b2a9b403, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.71666234Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
21952024-02-08T06:40:03.919Z{"msg":"Crucible stats registered with UUID: 3a0d6ae6-1b25-4e4d-833e-89b3b2a9b403","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716714776Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
21962024-02-08T06:40:03.919Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716739884Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215}
21972024-02-08T06:40:03.919Z{"msg":"3a0d6ae6-1b25-4e4d-833e-89b3b2a9b403 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716761042Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
21982024-02-08T06:40:03.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716800579Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
21992024-02-08T06:40:03.919Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716856655Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22002024-02-08T06:40:03.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716892922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22012024-02-08T06:40:03.919Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.71692323Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22022024-02-08T06:40:03.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716947628Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22032024-02-08T06:40:03.919Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.716970936Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22042024-02-08T06:40:03.919Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.717018313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22052024-02-08T06:40:03.919Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.717047121Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22062024-02-08T06:40:03.919Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.717083888Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22072024-02-08T06:40:03.919Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.717112746Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22082024-02-08T06:40:03.920Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.717152453Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22092024-02-08T06:40:03.920Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.71719181Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22102024-02-08T06:40:03.920Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.717697052Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22112024-02-08T06:40:03.920Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.717756217Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22122024-02-08T06:40:03.920Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.717803624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22132024-02-08T06:40:03.920Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.717839011Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22142024-02-08T06:40:03.920Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.718369391Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22152024-02-08T06:40:03.920Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.718413608Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22162024-02-08T06:40:03.920Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.718972296Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22172024-02-08T06:40:03.920Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.719011113Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22182024-02-08T06:40:03.920Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.719037971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22192024-02-08T06:40:03.920Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.719065679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22202024-02-08T06:40:03.920Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.719090347Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22212024-02-08T06:40:03.921Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.719117935Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22222024-02-08T06:40:03.921Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.719311301Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22232024-02-08T06:40:03.921Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.719361547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22242024-02-08T06:40:03.921Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.719788605Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22252024-02-08T06:40:03.921Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720225752Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22262024-02-08T06:40:03.921Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720263129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22272024-02-08T06:40:03.921Z{"msg":"[0]R flush_numbers[0..12]: [28, 23, 28, 28, 27, 20, 15, 23, 23, 29, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720279668Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22282024-02-08T06:40:03.921Z{"msg":"[0]R generation[0..12]: [18, 16, 18, 18, 17, 12, 8, 16, 16, 19, 19, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720321765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22292024-02-08T06:40:03.921Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720360972Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22302024-02-08T06:40:03.921Z{"msg":"[1]R flush_numbers[0..12]: [28, 23, 28, 28, 27, 20, 15, 23, 23, 29, 29, 28]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
22312024-02-08T06:40:03.921Z,"time":"2024-02-08T06:40:02.72038849Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22322024-02-08T06:40:03.921Z{"msg":"[1]R generation[0..12]: [18, 16, 18, 18, 17, 12, 8, 16, 16, 19, 19, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720414658Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22332024-02-08T06:40:03.921Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720444666Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22342024-02-08T06:40:03.922Z{"msg":"[2]R flush_numbers[0..12]: [28, 23, 28, 28, 27, 20, 15, 23, 23, 29, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720470284Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22352024-02-08T06:40:03.922Z{"msg":"[2]R generation[0..12]: [18, 16, 18, 18, 17, 12, 8, 16, 16, 19, 19, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720502631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22362024-02-08T06:40:03.922Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720529259Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22372024-02-08T06:40:03.922Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720559947Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22382024-02-08T06:40:03.922Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720583775Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22392024-02-08T06:40:03.922Z{"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720610313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22402024-02-08T06:40:03.922Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720633652Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22412024-02-08T06:40:03.923Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720660499Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22422024-02-08T06:40:03.923Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720682528Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22432024-02-08T06:40:03.923Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22442024-02-08T06:40:03.923Z,"time":"2024-02-08T06:40:02.720709696Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22452024-02-08T06:40:03.923Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720737024Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22462024-02-08T06:40:03.923Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720760092Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22472024-02-08T06:40:03.923Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.72078294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22482024-02-08T06:40:03.923Z{"msg":"3a0d6ae6-1b25-4e4d-833e-89b3b2a9b403 is now active with session: d79271a6-d45e-4be5-aee7-be03b30ab2eb","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720814328Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22492024-02-08T06:40:03.923Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720836526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215,"session_id":"d79271a6-d45e-4be5-aee7-be03b30ab2eb"}
22502024-02-08T06:40:03.924Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:02.720862364Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215}
22512024-02-08T06:40:03.924ZLoading write count information from file "/tmp/test_repair_verify.data"
22522024-02-08T06:40:03.924ZRun Repair workload
22532024-02-08T06:40:03.924Z01/30 Flush
22542024-02-08T06:40:03.924Z02/30 Write block 249 len 32768 data: 4 4 4 4 3 3 3 3
22552024-02-08T06:40:03.924Z03/30 Read block 84 len 24576
22562024-02-08T06:40:03.954Z04/30 Write block 211 len 16384 data: 3 4 4 4
22572024-02-08T06:40:03.954Z05/30 Flush
22582024-02-08T06:40:03.955Z06/30 Write block 90 len 12288 data: 4 5 6
22592024-02-08T06:40:03.955Z07/30 Read block 273 len 40960
22602024-02-08T06:40:03.955Z08/30 Read block 534 len 36864
22612024-02-08T06:40:03.955Z09/30 Read block 580 len 24576
22622024-02-08T06:40:03.955Z10/30 Read block 62 len 40960
22632024-02-08T06:40:03.955Z11/30 Write block 268 len 8192 data: 4 4
22642024-02-08T06:40:03.955Z12/30 Read block 347 len 8192
22652024-02-08T06:40:03.956Z13/30 Read block 496 len 28672
22662024-02-08T06:40:03.956Z14/30 Write block 436 len 32768 data: 2 2 2 2 2 2 2 2
22672024-02-08T06:40:03.956Z15/30 Read block 478 len 16384
22682024-02-08T06:40:03.956Z16/30 Flush
22692024-02-08T06:40:03.956Z17/30 Write block 170 len 12288 data: 3 3 3
22702024-02-08T06:40:03.956Z18/30 Write block 446 len 24576 data: 2 2 2 2 3 3
22712024-02-08T06:40:03.956Z19/30 Read block 150 len 12288
22722024-02-08T06:40:03.956Z20/30 Read block 433 len 28672
22732024-02-08T06:40:03.956Z21/30 Write block 298 len 36864 data: 4 6 5 4 4 3 3 3 2
22742024-02-08T06:40:03.957Z22/30 Write block 580 len 20480 data: 3 2 2 3 3
22752024-02-08T06:40:03.957Z23/30 Read block 580 len 40960
22762024-02-08T06:40:03.957Z24/30 Write block 50 len 32768 data: 3 3 3 3 4 4 5 5
22772024-02-08T06:40:03.957Z25/30 Read block 421 len 8192
22782024-02-08T06:40:03.958Z26/30 Write block 112 len 24576 data: 2 2 2 2 2 2
22792024-02-08T06:40:03.958Z27/30 Write block 68 len 28672 data: 2 2 2 2 2 2 2
22802024-02-08T06:40:03.959Z28/30 Read block 426 len 8192
22812024-02-08T06:40:03.959Z29/30 Write block 188 len 28672 data: 3 3 2 2 2 3 3
22822024-02-08T06:40:03.959Z30/30 Read block 347 len 24576
22832024-02-08T06:40:03.959Z----------------------------------------------------------------
22842024-02-08T06:40:03.959Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:29
22852024-02-08T06:40:03.959ZStates: Active Active Active
22862024-02-08T06:40:03.959ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22872024-02-08T06:40:03.960Z 2 Acked 1001 Write 8 Done Done Done false
22882024-02-08T06:40:03.960Z 3 Acked 1002 Read 6 Done Done Done false
22892024-02-08T06:40:03.960Z 4 Acked 1003 Write 4 Done Done Sent false
22902024-02-08T06:40:03.960Z 5 Acked 1004 Flush 0 Done Done Sent false
22912024-02-08T06:40:03.960Z 6 Acked 1005 Write 3 Done Done Sent false
22922024-02-08T06:40:03.960Z 7 Acked 1006 Read 10 Done Done Sent false
22932024-02-08T06:40:03.960Z 8 Acked 1007 Read 9 Done Done Sent false
22942024-02-08T06:40:03.960Z 9 Acked 1008 Read 6 Done Done Sent false
22952024-02-08T06:40:03.960Z 10 Acked 1009 Read 10 Done Done Sent false
22962024-02-08T06:40:03.960Z 11 Acked 1010 Write 2 Done Done Sent false
22972024-02-08T06:40:03.961Z 12 Acked 1011 Read 2 Done Done Sent false
22982024-02-08T06:40:03.961Z 13 Acked 1012 Read 7 Done Done Sent false
22992024-02-08T06:40:03.961Z 14 Acked 1013 Write 8 Done Done Sent false
23002024-02-08T06:40:03.961Z 15 Acked 1014 Read 4 Done Done Sent false
23012024-02-08T06:40:03.961Z 16 Acked 1015 Flush 0 Done Done Sent false
23022024-02-08T06:40:03.961Z 17 Acked 1016 Write 3 Done Done Sent false
23032024-02-08T06:40:03.961Z 18 Acked 1017 Write 6 Done Done Sent false
23042024-02-08T06:40:03.961Z 19 Acked 1018 Read 3 Done Done Sent false
23052024-02-08T06:40:03.961Z 20 Acked 1019 Read 7 Done Done Sent false
23062024-02-08T06:40:03.962Z 21 Acked 1020 Write 9 Done Done Sent false
23072024-02-08T06:40:03.962Z 22 Acked 1021 Write 5 Done Done Sent false
23082024-02-08T06:40:03.962Z 23 Acked 1022 Read 10 Done Done Sent false
23092024-02-08T06:40:03.962Z 24 Acked 1023 Write 8 Done Done Sent false
23102024-02-08T06:40:03.962Z 25 Acked 1024 Read 2 Done Done Sent false
23112024-02-08T06:40:03.962Z 26 Acked 1025 Write 6 Done Done Sent false
23122024-02-08T06:40:03.962Z 27 Acked 1026 Write 7 Done Done Sent false
23132024-02-08T06:40:03.962Z 28 Acked 1027 Read 2 Done Done Sent false
23142024-02-08T06:40:03.962Z 29 Acked 1028 Write 7 Done Done Sent false
23152024-02-08T06:40:03.962Z 30 Acked 1029 Read 6 Sent Done Sent false
23162024-02-08T06:40:03.962Z STATES DS:0 DS:1 DS:2 TOTAL
23172024-02-08T06:40:03.963Z New 0 0 0 0
23182024-02-08T06:40:03.963Z Sent 1 0 27 28
23192024-02-08T06:40:03.963Z Done 28 29 2 59
23202024-02-08T06:40:03.963Z Skipped 0 0 0 0
23212024-02-08T06:40:03.963Z Error 0 0 0 0
23222024-02-08T06:40:03.963ZLast Flush: 1015 1015 1000
23232024-02-08T06:40:03.963ZDownstairs last five completed: 1000
23242024-02-08T06:40:03.963ZUpstairs last five completed: 30 29 28 27 26
23252024-02-08T06:40:03.963Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:02.761909163Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1215}
23262024-02-08T06:40:03.963ZWrote out file "/tmp/test_repair_verify.data"
23272024-02-08T06:40:03.963Z
23282024-02-08T06:40:03.963ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23292024-02-08T06:40:03.963ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23302024-02-08T06:40:03.963Z 0 000-019 18 18 18 28 28 28 F F F
23312024-02-08T06:40:03.963Z 1 020-039 16 16 16 23 23 23 F F F
23322024-02-08T06:40:03.964Z 2 040-059 18 18 18 28 28 28 T T F <---
23332024-02-08T06:40:03.964Z 3 060-079 18 18 18 28 28 28 T T F <---
23342024-02-08T06:40:03.964Z 4 080-099 20 20 17 32 32 27 F F F <---
23352024-02-08T06:40:03.964Z 5 100-119 12 12 12 20 20 20 T T F <---
23362024-02-08T06:40:03.964Z 6 120-139 8 8 8 15 15 15 F F F
23372024-02-08T06:40:03.964Z 7 140-159 16 16 16 23 23 23 F F F
23382024-02-08T06:40:03.964Z 8 160-179 16 16 16 23 23 23 T T F <---
23392024-02-08T06:40:03.964Z 9 180-199 19 19 19 29 29 29 T T F <---
23402024-02-08T06:40:03.964Z 10 200-219 20 20 19 31 31 29 F F F <---
23412024-02-08T06:40:03.964Z 11 220-239 18 18 18 28 28 28 F F F
23422024-02-08T06:40:03.964Z 12 240-259 20 20 16 31 31 24 F F T <---
23432024-02-08T06:40:03.964Z 13 260-279 20 20 19 32 32 29 F F F <---
23442024-02-08T06:40:03.964Z 14 280-299 19 19 19 29 29 29 T T F <---
23452024-02-08T06:40:03.964Z 15 300-319 15 15 15 22 22 22 T T F <---
23462024-02-08T06:40:03.964Z 16 320-339 15 15 15 22 22 22 F F F
23472024-02-08T06:40:03.964Z 17 340-359 12 12 12 20 20 20 F F F
23482024-02-08T06:40:03.965Z 18 360-379 12 12 12 20 20 20 F F F
23492024-02-08T06:40:03.965Z 19 380-399 16 16 16 23 23 23 F F F
23502024-02-08T06:40:03.965Z 20 400-419 7 7 7 14 14 14 F F F
23512024-02-08T06:40:03.965Z 21 420-439 20 20 7 32 32 14 F F F <---
23522024-02-08T06:40:03.965Z 22 440-459 20 20 17 32 32 27 T T F <---
23532024-02-08T06:40:03.965Z 23 460-479 12 12 12 20 20 20 F F F
23542024-02-08T06:40:03.965Z 24 480-499 19 19 19 29 29 29 F F F
23552024-02-08T06:40:03.965Z 25 500-519 18 18 18 28 28 28 F F F
23562024-02-08T06:40:03.994Z 26 520-539 18 18 18 28 28 28 F F F
23572024-02-08T06:40:03.994Z 27 540-559 19 19 19 29 29 29 F F F
23582024-02-08T06:40:03.994Z 28 560-579 19 19 19 29 29 29 F F F
23592024-02-08T06:40:03.994Z 29 580-599 19 19 19 29 29 29 T T F <---
23602024-02-08T06:40:03.994ZMax gen: 20, Max flush: 32
23612024-02-08T06:40:03.995ZError: Difference in extent metadata found!
23622024-02-08T06:40:03.995ZA Difference in extent metadata is expected here
23632024-02-08T06:40:03.995ZOn loop 10
23642024-02-08T06:40:03.995Z
23652024-02-08T06:40:03.995ZVerifying data now
23662024-02-08T06:40:09.231ZNo change after verify
23672024-02-08T06:40:09.231ZLoop: 10 Downstairs dump after verify (and repair):
23682024-02-08T06:40:09.245ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23692024-02-08T06:40:09.246Z 0 000-019 18 18 18 28 28 28 F F F
23702024-02-08T06:40:09.249Z 1 020-039 16 16 16 23 23 23 F F F
23712024-02-08T06:40:09.249Z 2 040-059 21 21 21 33 33 33 F F F
23722024-02-08T06:40:09.250Z 3 060-079 21 21 21 33 33 33 F F F
23732024-02-08T06:40:09.250Z 4 080-099 20 20 20 32 32 32 F F F
23742024-02-08T06:40:09.250Z 5 100-119 21 21 21 33 33 33 F F F
23752024-02-08T06:40:09.250Z 6 120-139 8 8 8 15 15 15 F F F
23762024-02-08T06:40:09.250Z 7 140-159 16 16 16 23 23 23 F F F
23772024-02-08T06:40:09.250Z 8 160-179 21 21 21 33 33 33 F F F
23782024-02-08T06:40:09.250Z 9 180-199 21 21 21 33 33 33 F F F
23792024-02-08T06:40:09.250Z 10 200-219 20 20 20 31 31 31 F F F
23802024-02-08T06:40:09.250Z 11 220-239 18 18 18 28 28 28 F F F
23812024-02-08T06:40:09.250Z 12 240-259 20 20 20 31 31 31 F F F
23822024-02-08T06:40:09.250Z 13 260-279 20 20 20 32 32 32 F F F
23832024-02-08T06:40:09.250Z 14 280-299 21 21 21 33 33 33 F F F
23842024-02-08T06:40:09.250Z 15 300-319 21 21 21 33 33 33 F F F
23852024-02-08T06:40:09.250Z 16 320-339 15 15 15 22 22 22 F F F
23862024-02-08T06:40:09.250Z 17 340-359 12 12 12 20 20 20 F F F
23872024-02-08T06:40:09.251Z 18 360-379 12 12 12 20 20 20 F F F
23882024-02-08T06:40:09.251Z 19 380-399 16 16 16 23 23 23 F F F
23892024-02-08T06:40:09.251Z 20 400-419 7 7 7 14 14 14 F F F
23902024-02-08T06:40:09.251Z 21 420-439 20 20 20 32 32 32 F F F
23912024-02-08T06:40:09.251Z 22 440-459 21 21 21 33 33 33 F F F
23922024-02-08T06:40:09.251Z 23 460-479 12 12 12 20 20 20 F F F
23932024-02-08T06:40:09.251Z 24 480-499 19 19 19 29 29 29 F F F
23942024-02-08T06:40:09.251Z 25 500-519 18 18 18 28 28 28 F F F
23952024-02-08T06:40:09.251Z 26 520-539 18 18 18 28 28 28 F F F
23962024-02-08T06:40:09.251Z 27 540-559 19 19 19 29 29 29 F F F
23972024-02-08T06:40:09.251Z 28 560-579 19 19 19 29 29 29 F F F
23982024-02-08T06:40:09.251Z 29 580-599 21 21 21 33 33 33 F F F
23992024-02-08T06:40:09.251ZMax gen: 21, Max flush: 33
24002024-02-08T06:40:09.251Z
24012024-02-08T06:40:09.251ZBegin loop 11
24022024-02-08T06:40:09.252ZDownstairs to restart: 2
24032024-02-08T06:40:09.267Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.519217326Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223}
24042024-02-08T06:40:09.267Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\nCrucible runtime is spawned
24052024-02-08T06:40:09.273Z rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.519538432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223}
24062024-02-08T06:40:09.273Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.51957258Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223}
24072024-02-08T06:40:09.273Z{"msg":"Crucible 7ff9fb44-2e59-461d-ba9b-0b691a4c9008 has session id: c64e34af-35d2-431f-a296-83469455b1fe","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.519617786Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24082024-02-08T06:40:09.273Z{"msg":"Upstairs opts: Upstairs UUID: 7ff9fb44-2e59-461d-ba9b-0b691a4c9008, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.519654623Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24092024-02-08T06:40:09.273Z{"msg":"Crucible stats registered with UUID: 7ff9fb44-2e59-461d-ba9b-0b691a4c9008","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.519693191Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24102024-02-08T06:40:09.273Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.519728148Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223}
24112024-02-08T06:40:09.274Z{"msg":"7ff9fb44-2e59-461d-ba9b-0b691a4c9008 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.519766915Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24122024-02-08T06:40:09.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.519819261Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24132024-02-08T06:40:09.274Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.519873657Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24142024-02-08T06:40:09.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.519917414Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24152024-02-08T06:40:09.274Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.51995988Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24162024-02-08T06:40:09.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.520016286Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24172024-02-08T06:40:09.274Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.5201017Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24182024-02-08T06:40:09.274Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.520154856Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24192024-02-08T06:40:09.274Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.520182014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24202024-02-08T06:40:09.274Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.520225911Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24212024-02-08T06:40:09.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.520272997Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24222024-02-08T06:40:09.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.520342012Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24232024-02-08T06:40:09.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.520379559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24242024-02-08T06:40:09.274Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.52090095Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24252024-02-08T06:40:09.274Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.520948016Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24262024-02-08T06:40:09.274Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.520979354Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24272024-02-08T06:40:09.275Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.52103798Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24282024-02-08T06:40:09.275Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.521473227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24292024-02-08T06:40:09.275Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.521645884Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24302024-02-08T06:40:09.275Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.522006827Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24312024-02-08T06:40:09.275Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.522056403Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24322024-02-08T06:40:09.275Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.522084711Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24332024-02-08T06:40:09.275Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.522121448Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24342024-02-08T06:40:09.275Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.522143636Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24352024-02-08T06:40:09.275Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.522165495Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24362024-02-08T06:40:09.275Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.522178524Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24372024-02-08T06:40:09.275Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.522192233Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24382024-02-08T06:40:09.275Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.522647979Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24392024-02-08T06:40:09.280Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523113464Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24402024-02-08T06:40:09.280Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523155161Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24412024-02-08T06:40:09.280Z{"msg":"[0]R flush_numbers[0..12]: [28, 23, 33, 33, 32, 33, 15, 23, 33, 33, 31, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523183049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24422024-02-08T06:40:09.281Z{"msg":"[0]R generation[0..12]: [18, 16, 21, 21, 20, 21, 8, 16, 21, 21, 20, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523213896Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24432024-02-08T06:40:09.281Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523238724Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24442024-02-08T06:40:09.281Z{Wait for a query_work_queue command to finish before sending IO
24452024-02-08T06:40:09.281Z"msg":"[1]R flush_numbers[0..12]: [28, 23, 33, 33, 32, 33, 15, 23, 33, 33, 31, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.52330138Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24462024-02-08T06:40:09.281Z{"msg":"[1]R generation[0..12]: [18, 16, 21, 21, 20, 21, 8, 16, 21, 21, 20, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523340687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24472024-02-08T06:40:09.281Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523358505Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24482024-02-08T06:40:09.281Z{"msg":"[2]R flush_numbers[0..12]: [28, 23, 33, 33, 32, 33, 15, 23, 33, 33, 31, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523381544Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24492024-02-08T06:40:09.281Z{"msg":"[2]R generation[0..12]: [18, 16, 21, 21, 20, 21, 8, 16, 21, 21, 20, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523396913Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24502024-02-08T06:40:09.281Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523419411Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24512024-02-08T06:40:09.281Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.5234336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24522024-02-08T06:40:09.281Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523454638Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24532024-02-08T06:40:09.281Z{"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523468297Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24542024-02-08T06:40:09.281Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523489135Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24552024-02-08T06:40:09.281Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523503074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24562024-02-08T06:40:09.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523522463Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24572024-02-08T06:40:09.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523536472Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24582024-02-08T06:40:09.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523557221Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24592024-02-08T06:40:09.282Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523571129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24602024-02-08T06:40:09.282Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523583708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24612024-02-08T06:40:09.282Z{"msg":"7ff9fb44-2e59-461d-ba9b-0b691a4c9008 is now active with session: c64e34af-35d2-431f-a296-83469455b1fe","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523603827Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24622024-02-08T06:40:09.282Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523616776Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223,"session_id":"c64e34af-35d2-431f-a296-83469455b1fe"}
24632024-02-08T06:40:09.282Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:08.523629845Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223}
24642024-02-08T06:40:09.282ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24652024-02-08T06:40:09.282ZLoading write count information from file "/tmp/test_repair_verify.data"
24662024-02-08T06:40:09.282ZRun Repair workload
24672024-02-08T06:40:09.282Z01/30 Write block 285 len 36864 data: 2 2 5 4 4 6 6 4 3
24682024-02-08T06:40:09.282Z02/30 Write block 291 len 16384 data: 7 5 4 3
24692024-02-08T06:40:09.282Z03/30 Flush
24702024-02-08T06:40:09.282Z04/30 Write block 42 len 20480 data: 3 3 3 2 3
24712024-02-08T06:40:09.282Z05/30 Read block 504 len 20480
24722024-02-08T06:40:09.282Z06/30 Write block 319 len 20480 data: 5 5 3 3 3
24732024-02-08T06:40:09.283Z07/30 Write block 106 len 40960 data: 4 4 4 4 2 2 3 3 3 3
24742024-02-08T06:40:09.286Z08/30 Read block 380 len 4096
24752024-02-08T06:40:09.286Z09/30 Read block 353 len 36864
24762024-02-08T06:40:09.286Z10/30 Read block 138 len 40960
24772024-02-08T06:40:09.286Z11/30 Write block 242 len 24576 data: 2 2 2 2 2 3
24782024-02-08T06:40:09.286Z12/30 Write block 219 len 36864 data: 2 2 3 4 4 5 5 6 6
24792024-02-08T06:40:09.286Z13/30 Read block 88 len 36864
24802024-02-08T06:40:09.286Z14/30 Flush
24812024-02-08T06:40:09.290Z15/30 Write block 420 len 4096 data: 2
24822024-02-08T06:40:09.290Z16/30 Write block 344 len 32768 data: 3 4 4 4 3 4 5 5
24832024-02-08T06:40:09.290Z17/30 Read block 260 len 16384
24842024-02-08T06:40:09.293Z18/30 Read block 117 len 8192
24852024-02-08T06:40:09.293Z19/30 Write block 241 len 4096 data: 2
24862024-02-08T06:40:09.293Z20/30 Read block 375 len 20480
24872024-02-08T06:40:09.294Z21/30 Read block 25 len 20480
24882024-02-08T06:40:09.294Z22/30 Read block 103 len 40960
24892024-02-08T06:40:09.300Z23/30 Write block 22 len 36864 data: 2 2 3 4 4 4 3 2 3
24902024-02-08T06:40:09.300Z24/30 Write block 551 len 24576 data: 3 3 4 4 3 4
24912024-02-08T06:40:09.300Z25/30 Write block 536 len 24576 data: 3 3 2 2 2 2
24922024-02-08T06:40:09.300Z26/30 Read block 193 len 8192
24932024-02-08T06:40:09.300Z27/30 Write block 175 len 36864 data: 2 2 2 3 3 2 3 3 3
24942024-02-08T06:40:09.300Z28/30 Read block 83 len 24576
24952024-02-08T06:40:09.307Z29/30 Write block 13 len 4096 data: 3
24962024-02-08T06:40:09.308Z30/30 Read block 145 len 4096
24972024-02-08T06:40:09.308Z----------------------------------------------------------------
24982024-02-08T06:40:09.308Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:27
24992024-02-08T06:40:09.308ZStates: Active Active Active
25002024-02-08T06:40:09.308ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
25012024-02-08T06:40:09.308Z 4 Acked 1003 Write 5 Done Done Sent false
25022024-02-08T06:40:09.308Z 5 Acked 1004 Read 5 Done Done Sent false
25032024-02-08T06:40:09.308Z 6 Acked 1005 Write 5 Done Done Sent false
25042024-02-08T06:40:09.308Z 7 Acked 1006 Write 10 Done Done Sent false
25052024-02-08T06:40:09.308Z 8 Acked 1007 Read 1 Done Done Sent false
25062024-02-08T06:40:09.308Z 9 Acked 1008 Read 9 Done Done Sent false
25072024-02-08T06:40:09.308Z 10 Acked 1009 Read 10 Done Done Sent false
25082024-02-08T06:40:09.308Z 11 Acked 1010 Write 6 Done Done Sent false
25092024-02-08T06:40:09.308Z 12 Acked 1011 Write 9 Done Done Sent false
25102024-02-08T06:40:09.308Z 13 Acked 1012 Read 9 Done Done Sent false
25112024-02-08T06:40:09.309Z 14 Acked 1013 Flush 0 Done Done Sent false
25122024-02-08T06:40:09.309Z 15 Acked 1014 Write 1 Done Done Sent false
25132024-02-08T06:40:09.309Z 16 Acked 1015 Write 8 Done Done Sent false
25142024-02-08T06:40:09.309Z 17 Acked 1016 Read 4 Done Done Sent false
25152024-02-08T06:40:09.309Z 18 Acked 1017 Read 2 Done Done Sent false
25162024-02-08T06:40:09.309Z 19 Acked 1018 Write 1 Done Done Sent false
25172024-02-08T06:40:09.309Z 20 Acked 1019 Read 5 Done Done Sent false
25182024-02-08T06:40:09.309Z 21 Acked 1020 Read 5 Done Done Sent false
25192024-02-08T06:40:09.309Z 22 Acked 1021 Read 10 Done Done Sent false
25202024-02-08T06:40:09.309Z 23 Acked 1022 Write 9 Done Done Sent false
25212024-02-08T06:40:09.309Z 24 Acked 1023 Write 6 Done Done Sent false
25222024-02-08T06:40:09.309Z 25 Acked 1024 Write 6 Done Done Sent false
25232024-02-08T06:40:09.309Z 26 Acked 1025 Read 2 Done Done Sent false
25242024-02-08T06:40:09.309Z 27 Acked 1026 Write 9 Done Done Sent false
25252024-02-08T06:40:09.309Z 28 Acked 1027 Read 6 Done Done Sent false
25262024-02-08T06:40:09.310Z 29 Acked 1028 Write 1 Done Done Sent false
25272024-02-08T06:40:09.310Z 30 Acked 1029 Read 1 Done Done Sent false
25282024-02-08T06:40:09.310Z STATES DS:0 DS:1 DS:2 TOTAL
25292024-02-08T06:40:09.310Z New 0 0 0 0
25302024-02-08T06:40:09.310Z Sent 0 0 27 27
25312024-02-08T06:40:09.310Z Done 27 27 0 54
25322024-02-08T06:40:09.310Z Skipped 0 0 0 0
25332024-02-08T06:40:09.310Z Error 0 0 0 0
25342024-02-08T06:40:09.310ZLast Flush: 1013 1013 1002
25352024-02-08T06:40:09.310ZDownstairs last five completed: 1002 1001 1000
25362024-02-08T06:40:09.310ZUpstairs last five completed: 30 29 28 27 26
25372024-02-08T06:40:09.310Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:08.553186696Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1223}
25382024-02-08T06:40:09.310ZWrote out file "/tmp/test_repair_verify.data"
25392024-02-08T06:40:09.314Z
25402024-02-08T06:40:09.314ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25412024-02-08T06:40:09.326ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25422024-02-08T06:40:09.326Z 0 000-019 18 18 18 28 28 28 T T F <---
25432024-02-08T06:40:09.326Z 1 020-039 16 16 16 23 23 23 T T F <---
25442024-02-08T06:40:09.326Z 2 040-059 22 22 21 35 35 33 F F F <---
25452024-02-08T06:40:09.326Z 3 060-079 21 21 21 33 33 33 F F F
25462024-02-08T06:40:09.326Z 4 080-099 20 20 20 32 32 32 F F F
25472024-02-08T06:40:09.326Z 5 100-119 22 22 21 35 35 33 F F F <---
25482024-02-08T06:40:09.326Z 6 120-139 8 8 8 15 15 15 F F F
25492024-02-08T06:40:09.327Z 7 140-159 16 16 16 23 23 23 F F F
25502024-02-08T06:40:09.327Z 8 160-179 21 21 21 33 33 33 T T F <---
25512024-02-08T06:40:09.327Z 9 180-199 21 21 21 33 33 33 T T F <---
25522024-02-08T06:40:09.327Z 10 200-219 22 22 20 35 35 31 F F F <---
25532024-02-08T06:40:09.330Z 11 220-239 22 22 18 35 35 28 F F F <---
25542024-02-08T06:40:09.330Z 12 240-259 22 22 20 35 35 31 T T F <---
25552024-02-08T06:40:09.330Z 13 260-279 20 20 20 32 32 32 F F F
25562024-02-08T06:40:09.331Z 14 280-299 22 22 22 34 34 34 F F F
25572024-02-08T06:40:09.331Z 15 300-319 22 22 21 35 35 33 F F F <---
25582024-02-08T06:40:09.331Z 16 320-339 22 22 15 35 35 22 F F F <---
25592024-02-08T06:40:09.331Z 17 340-359 12 12 12 20 20 20 T T F <---
25602024-02-08T06:40:09.331Z 18 360-379 12 12 12 20 20 20 F F F
25612024-02-08T06:40:09.331Z 19 380-399 16 16 16 23 23 23 F F F
25622024-02-08T06:40:09.331Z 20 400-419 7 7 7 14 14 14 F F F
25632024-02-08T06:40:09.331Z 21 420-439 20 20 20 32 32 32 T T F <---
25642024-02-08T06:40:09.331Z 22 440-459 21 21 21 33 33 33 F F F
25652024-02-08T06:40:09.331Z 23 460-479 12 12 12 20 20 20 F F F
25662024-02-08T06:40:09.331Z 24 480-499 19 19 19 29 29 29 F F F
25672024-02-08T06:40:09.331Z 25 500-519 18 18 18 28 28 28 F F F
25682024-02-08T06:40:09.331Z 26 520-539 18 18 18 28 28 28 T T F <---
25692024-02-08T06:40:09.331Z 27 540-559 19 19 19 29 29 29 T T F <---
25702024-02-08T06:40:09.331Z 28 560-579 19 19 19 29 29 29 F F F
25712024-02-08T06:40:09.332Z 29 580-599 21 21 21 33 33 33 F F F
25722024-02-08T06:40:09.332ZMax gen: 22, Max flush: 35
25732024-02-08T06:40:09.332ZError: Difference in extent metadata found!
25742024-02-08T06:40:09.332ZA Difference in extent metadata is expected here
25752024-02-08T06:40:09.332ZOn loop 11
25762024-02-08T06:40:09.332Z
25772024-02-08T06:40:09.335ZVerifying data now
25782024-02-08T06:40:14.038ZNo change after verify
25792024-02-08T06:40:14.038ZLoop: 11 Downstairs dump after verify (and repair):
25802024-02-08T06:40:14.051ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25812024-02-08T06:40:14.051Z 0 000-019 23 23 23 36 36 36 F F F
25822024-02-08T06:40:14.051Z 1 020-039 23 23 23 36 36 36 F F F
25832024-02-08T06:40:14.051Z 2 040-059 22 22 22 35 35 35 F F F
25842024-02-08T06:40:14.055Z 3 060-079 21 21 21 33 33 33 F F F
25852024-02-08T06:40:14.055Z 4 080-099 20 20 20 32 32 32 F F F
25862024-02-08T06:40:14.055Z 5 100-119 22 22 22 35 35 35 F F F
25872024-02-08T06:40:14.055Z 6 120-139 8 8 8 15 15 15 F F F
25882024-02-08T06:40:14.055Z 7 140-159 16 16 16 23 23 23 F F F
25892024-02-08T06:40:14.055Z 8 160-179 23 23 23 36 36 36 F F F
25902024-02-08T06:40:14.055Z 9 180-199 23 23 23 36 36 36 F F F
25912024-02-08T06:40:14.055Z 10 200-219 22 22 22 35 35 35 F F F
25922024-02-08T06:40:14.055Z 11 220-239 22 22 22 35 35 35 F F F
25932024-02-08T06:40:14.055Z 12 240-259 23 23 23 36 36 36 F F F
25942024-02-08T06:40:14.055Z 13 260-279 20 20 20 32 32 32 F F F
25952024-02-08T06:40:14.055Z 14 280-299 22 22 22 34 34 34 F F F
25962024-02-08T06:40:14.055Z 15 300-319 22 22 22 35 35 35 F F F
25972024-02-08T06:40:14.055Z 16 320-339 22 22 22 35 35 35 F F F
25982024-02-08T06:40:14.056Z 17 340-359 23 23 23 36 36 36 F F F
25992024-02-08T06:40:14.056Z 18 360-379 12 12 12 20 20 20 F F F
26002024-02-08T06:40:14.056Z 19 380-399 16 16 16 23 23 23 F F F
26012024-02-08T06:40:14.056Z 20 400-419 7 7 7 14 14 14 F F F
26022024-02-08T06:40:14.056Z 21 420-439 23 23 23 36 36 36 F F F
26032024-02-08T06:40:14.056Z 22 440-459 21 21 21 33 33 33 F F F
26042024-02-08T06:40:14.056Z 23 460-479 12 12 12 20 20 20 F F F
26052024-02-08T06:40:14.056Z 24 480-499 19 19 19 29 29 29 F F F
26062024-02-08T06:40:14.056Z 25 500-519 18 18 18 28 28 28 F F F
26072024-02-08T06:40:14.056Z 26 520-539 23 23 23 36 36 36 F F F
26082024-02-08T06:40:14.056Z 27 540-559 23 23 23 36 36 36 F F F
26092024-02-08T06:40:14.056Z 28 560-579 19 19 19 29 29 29 F F F
26102024-02-08T06:40:14.056Z 29 580-599 21 21 21 33 33 33 F F F
26112024-02-08T06:40:14.056ZMax gen: 23, Max flush: 36
26122024-02-08T06:40:14.056Z
26132024-02-08T06:40:14.056ZBegin loop 12
26142024-02-08T06:40:14.057ZDownstairs to restart: 1
26152024-02-08T06:40:14.074Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325310196Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231}
26162024-02-08T06:40:14.074Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325578116Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231}
26172024-02-08T06:40:14.078Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325601044Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231}
26182024-02-08T06:40:14.078Z{"msg":"Crucible 929e4f0a-397a-4db3-83fc-f02c3dfcff9b has session id: 3a5cd7c5-9ec0-459a-ba50-496a5556cd9e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325633532Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26192024-02-08T06:40:14.078Z{"msg":"Upstairs opts: Upstairs UUID: 929e4f0a-397a-4db3-83fc-f02c3dfcff9b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325665599Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26202024-02-08T06:40:14.079Z{"msg":"Crucible stats registered with UUID: 929e4f0a-397a-4db3-83fc-f02c3dfcff9b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325703497Z",Crucible runtime is spawned
26212024-02-08T06:40:14.079Z"hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26222024-02-08T06:40:14.079Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325799809Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231}
26232024-02-08T06:40:14.079Z{"msg":"929e4f0a-397a-4db3-83fc-f02c3dfcff9b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325889813Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26242024-02-08T06:40:14.079Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325917391Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26252024-02-08T06:40:14.079Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325941329Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26262024-02-08T06:40:14.079Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.325977566Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26272024-02-08T06:40:14.079Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.326008064Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26282024-02-08T06:40:14.079Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.326035102Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26292024-02-08T06:40:14.079Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.32605977Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26302024-02-08T06:40:14.079Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.326087508Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26312024-02-08T06:40:14.079Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.326114406Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26322024-02-08T06:40:14.079Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.326140154Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26332024-02-08T06:40:14.079Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.326222048Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26342024-02-08T06:40:14.079Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.326282753Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26352024-02-08T06:40:14.080Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.326311111Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26362024-02-08T06:40:14.080Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.326916416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26372024-02-08T06:40:14.080Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.326964042Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26382024-02-08T06:40:14.080Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.327055025Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26392024-02-08T06:40:14.080Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.327105731Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26402024-02-08T06:40:14.080Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.327573356Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26412024-02-08T06:40:14.080Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.327707226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26422024-02-08T06:40:14.080Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.32818733Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26432024-02-08T06:40:14.080Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.328225067Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26442024-02-08T06:40:14.080Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.328247696Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26452024-02-08T06:40:14.080Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.328273634Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26462024-02-08T06:40:14.080Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.328300422Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26472024-02-08T06:40:14.080Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.32831642Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26482024-02-08T06:40:14.080Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.328332429Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26492024-02-08T06:40:14.081Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.328346128Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26502024-02-08T06:40:14.085Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.328828372Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26512024-02-08T06:40:14.085Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329273499Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26522024-02-08T06:40:14.085Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329309046Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26532024-02-08T06:40:14.086Z{"msg":"[0]R flush_numbers[0..12]: [36, 36, 35, 33, 32, 35, 15, 23, 36, 36, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329325475Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26542024-02-08T06:40:14.086Z{"msg":"[0]R generation[0..12]: [23, 23, 22, 21, 20, 22, 8, 16, 23, 23, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329340874Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26552024-02-08T06:40:14.086Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329362902Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26562024-02-08T06:40:14.086Z{"msg":"[1]R flush_numbers[0..12]: [36, 36, 35, 33, 32, 35, 15, 23, 36, 36, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329377871Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26572024-02-08T06:40:14.086Z{"msg":"[1]R generation[0..12]: [23, 23, 22, 21, 20, 22, 8, 16, 23, 23, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329399079Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26582024-02-08T06:40:14.086Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329413708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26592024-02-08T06:40:14.086Z{"msg":"[2]R flush_numbers[0..12]: [36, 36, 35, 33, 32, 35, 15, 23, 36, 36, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329434457Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26602024-02-08T06:40:14.086Z{"msg":"[2]R generation[0..12]: [23, 23, 22, 21, 20, 22, 8, 16, 23, 23, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329449566Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26612024-02-08T06:40:14.086Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329463794Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26622024-02-08T06:40:14.086Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329477323Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26632024-02-08T06:40:14.086Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329498422Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
26642024-02-08T06:40:14.086Z3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26652024-02-08T06:40:14.086Z{"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.32951523Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26662024-02-08T06:40:14.086Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329535309Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26672024-02-08T06:40:14.086Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329549948Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26682024-02-08T06:40:14.087Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329614233Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26692024-02-08T06:40:14.087Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.3296506Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26702024-02-08T06:40:14.087Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329692897Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26712024-02-08T06:40:14.087Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329723255Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26722024-02-08T06:40:14.087Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329738694Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26732024-02-08T06:40:14.087Z{"msg":"929e4f0a-397a-4db3-83fc-f02c3dfcff9b is now active with session: 3a5cd7c5-9ec0-459a-ba50-496a5556cd9e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329769031Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26742024-02-08T06:40:14.087Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.32978512Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231,"session_id":"3a5cd7c5-9ec0-459a-ba50-496a5556cd9e"}
26752024-02-08T06:40:14.087Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:13.329813728Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231}
26762024-02-08T06:40:14.087ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26772024-02-08T06:40:14.087ZLoading write count information from file "/tmp/test_repair_verify.data"
26782024-02-08T06:40:14.087ZRun Repair workload
26792024-02-08T06:40:14.087Z01/30 Write block 61 len 36864 data: 2 2 2 2 2 2 2 3 3
26802024-02-08T06:40:14.087Z02/30 Write block 90 len 28672 data: 5 6 7 6 6 6 4
26812024-02-08T06:40:14.087Z03/30 Read block 131 len 40960
26822024-02-08T06:40:14.088Z04/30 Flush
26832024-02-08T06:40:14.091Z05/30 Write block 417 len 36864 data: 3 3 2 3 2 3 3 3 2
26842024-02-08T06:40:14.091Z06/30 Write block 128 len 4096 data: 2
26852024-02-08T06:40:14.091Z07/30 Read block 69 len 32768
26862024-02-08T06:40:14.091Z08/30 Write block 368 len 24576 data: 2 3 2 2 2 2
26872024-02-08T06:40:14.091Z09/30 Write block 137 len 12288 data: 6 4 4
26882024-02-08T06:40:14.091Z10/30 Write block 434 len 16384 data: 2 2 3 3
26892024-02-08T06:40:14.091Z11/30 Read block 153 len 16384
26902024-02-08T06:40:14.091Z12/30 Read block 119 len 40960
26912024-02-08T06:40:14.104Z13/30 Write block 198 len 8192 data: 4 4
26922024-02-08T06:40:14.104Z14/30 Read block 35 len 20480
26932024-02-08T06:40:14.104Z15/30 Read block 140 len 4096
26942024-02-08T06:40:14.104Z16/30 Write block 360 len 12288 data: 2 2 2
26952024-02-08T06:40:14.104Z17/30 Read block 162 len 32768
26962024-02-08T06:40:14.107Z18/30 Read block 197 len 24576
26972024-02-08T06:40:14.107Z19/30 Write block 402 len 20480 data: 2 2 2 2 2
26982024-02-08T06:40:14.108Z20/30 Read block 151 len 32768
26992024-02-08T06:40:14.108Z21/30 Write block 235 len 36864 data: 4 4 3 3 3 2 3 3 3
27002024-02-08T06:40:14.108Z22/30 Read block 430 len 28672
27012024-02-08T06:40:14.108Z23/30 Read block 475 len 36864
27022024-02-08T06:40:14.108Z24/30 Read block 549 len 24576
27032024-02-08T06:40:14.108Z25/30 Flush
27042024-02-08T06:40:14.108Z26/30 Flush
27052024-02-08T06:40:14.108Z27/30 Write block 203 len 28672 data: 3 3 3 3 3 3 3
27062024-02-08T06:40:14.108Z28/30 Read block 375 len 28672
27072024-02-08T06:40:14.108Z29/30 Read block 113 len 16384
27082024-02-08T06:40:14.112Z30/30 Write block 243 len 36864 data: 4 3 3 3 4 3 5 5 5
27092024-02-08T06:40:14.112Z----------------------------------------------------------------
27102024-02-08T06:40:14.112Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
27112024-02-08T06:40:14.112ZStates: Active Active Active
27122024-02-08T06:40:14.112ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
27132024-02-08T06:40:14.112Z 1 Acked 1000 Write 9 Done Done Done false
27142024-02-08T06:40:14.112Z 2 Acked 1001 Write 7 Done Done Done false
27152024-02-08T06:40:14.112Z 3 Acked 1002 Read 10 Done Done Done false
27162024-02-08T06:40:14.112Z 4 Acked 1003 Flush 0 Done Sent Done false
27172024-02-08T06:40:14.112Z 5 Acked 1004 Write 9 Done Sent Done false
27182024-02-08T06:40:14.112Z 6 Acked 1005 Write 1 Done Sent Done false
27192024-02-08T06:40:14.113Z 7 Acked 1006 Read 8 Done Sent Done false
27202024-02-08T06:40:14.113Z 8 Acked 1007 Write 6 Done Sent Done false
27212024-02-08T06:40:14.113Z 9 Acked 1008 Write 3 Done Sent Done false
27222024-02-08T06:40:14.113Z 10 Acked 1009 Write 4 Done Sent Done false
27232024-02-08T06:40:14.113Z 11 Acked 1010 Read 4 Done Sent Done false
27242024-02-08T06:40:14.113Z 12 Acked 1011 Read 10 Done Sent Done false
27252024-02-08T06:40:14.113Z 13 Acked 1012 Write 2 Done Sent Done false
27262024-02-08T06:40:14.113Z 14 Acked 1013 Read 5 Done Sent Done false
27272024-02-08T06:40:14.113Z 15 Acked 1014 Read 1 Done Sent Done false
27282024-02-08T06:40:14.113Z 16 Acked 1015 Write 3 Done Sent Done false
27292024-02-08T06:40:14.113Z 17 Acked 1016 Read 8 Done Sent Done false
27302024-02-08T06:40:14.113Z 18 Acked 1017 Read 6 Done Sent Done false
27312024-02-08T06:40:14.113Z 19 Acked 1018 Write 5 Done Sent Done false
27322024-02-08T06:40:14.113Z 20 Acked 1019 Read 8 Done Sent Done false
27332024-02-08T06:40:14.113Z 21 Acked 1020 Write 9 Done Sent Done false
27342024-02-08T06:40:14.114Z 22 Acked 1021 Read 7 Done Sent Done false
27352024-02-08T06:40:14.114Z 23 Acked 1022 Read 9 Done Sent Done false
27362024-02-08T06:40:14.114Z 24 Acked 1023 Read 6 Done Sent Done false
27372024-02-08T06:40:14.114Z 25 Acked 1024 Flush 0 Done Sent Done false
27382024-02-08T06:40:14.114Z 26 Acked 1025 Flush 0 Done Sent Done false
27392024-02-08T06:40:14.114Z 27 Acked 1026 Write 7 Done Sent Done false
27402024-02-08T06:40:14.114Z 28 Acked 1027 Read 7 Done Sent Done false
27412024-02-08T06:40:14.114Z 29 Acked 1028 Read 4 Done Sent Done false
27422024-02-08T06:40:14.114Z 30 Acked 1029 Write 9 Sent Sent Sent false
27432024-02-08T06:40:14.114Z STATES DS:0 DS:1 DS:2 TOTAL
27442024-02-08T06:40:14.114Z New 0 0 0 0
27452024-02-08T06:40:14.114Z Sent 1 27 1 29
27462024-02-08T06:40:14.114Z Done 29 3 29 61
27472024-02-08T06:40:14.114Z Skipped 0 0 0 0
27482024-02-08T06:40:14.114Z Error 0 0 0 0
27492024-02-08T06:40:14.114ZLast Flush: 1025 0 1025
27502024-02-08T06:40:14.115ZDownstairs last five completed:
27512024-02-08T06:40:14.115ZUpstairs last five completed: 30 29 28 27 26
27522024-02-08T06:40:14.115Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:13.360623775Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1231}
27532024-02-08T06:40:14.115ZWrote out file "/tmp/test_repair_verify.data"
27542024-02-08T06:40:14.118Z
27552024-02-08T06:40:14.118ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27562024-02-08T06:40:14.132ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27572024-02-08T06:40:14.132Z 0 000-019 23 23 23 36 36 36 F F F
27582024-02-08T06:40:14.132Z 1 020-039 23 23 23 36 36 36 F F F
27592024-02-08T06:40:14.132Z 2 040-059 22 22 22 35 35 35 F F F
27602024-02-08T06:40:14.132Z 3 060-079 24 21 24 37 33 37 F T F <---
27612024-02-08T06:40:14.132Z 4 080-099 24 20 24 37 32 37 F T F <---
27622024-02-08T06:40:14.132Z 5 100-119 22 22 22 35 35 35 F F F
27632024-02-08T06:40:14.132Z 6 120-139 24 8 24 38 15 38 F F F <---
27642024-02-08T06:40:14.132Z 7 140-159 16 16 16 23 23 23 F F F
27652024-02-08T06:40:14.132Z 8 160-179 23 23 23 36 36 36 F F F
27662024-02-08T06:40:14.132Z 9 180-199 24 23 24 38 36 38 F F F <---
27672024-02-08T06:40:14.136Z 10 200-219 22 22 22 35 35 35 T F T <---
27682024-02-08T06:40:14.136Z 11 220-239 24 22 24 38 35 38 F F F <---
27692024-02-08T06:40:14.136Z 12 240-259 24 23 24 38 36 38 T F T <---
27702024-02-08T06:40:14.136Z 13 260-279 20 20 20 32 32 32 F F F
27712024-02-08T06:40:14.136Z 14 280-299 22 22 22 34 34 34 F F F
27722024-02-08T06:40:14.136Z 15 300-319 22 22 22 35 35 35 F F F
27732024-02-08T06:40:14.136Z 16 320-339 22 22 22 35 35 35 F F F
27742024-02-08T06:40:14.136Z 17 340-359 23 23 23 36 36 36 F F F
27752024-02-08T06:40:14.136Z 18 360-379 24 12 24 38 20 38 F F F <---
27762024-02-08T06:40:14.136Z 19 380-399 16 16 16 23 23 23 F F F
27772024-02-08T06:40:14.136Z 20 400-419 24 7 24 38 14 38 F F F <---
27782024-02-08T06:40:14.136Z 21 420-439 24 23 24 38 36 38 F F F <---
27792024-02-08T06:40:14.136Z 22 440-459 21 21 21 33 33 33 F F F
27802024-02-08T06:40:14.136Z 23 460-479 12 12 12 20 20 20 F F F
27812024-02-08T06:40:14.137Z 24 480-499 19 19 19 29 29 29 F F F
27822024-02-08T06:40:14.137Z 25 500-519 18 18 18 28 28 28 F F F
27832024-02-08T06:40:14.137Z 26 520-539 23 23 23 36 36 36 F F F
27842024-02-08T06:40:14.137Z 27 540-559 23 23 23 36 36 36 F F F
27852024-02-08T06:40:14.137Z 28 560-579 19 19 19 29 29 29 F F F
27862024-02-08T06:40:14.137Z 29 580-599 21 21 21 33 33 33 F F F
27872024-02-08T06:40:14.137ZMax gen: 24, Max flush: 38
27882024-02-08T06:40:14.137ZError: Difference in extent metadata found!
27892024-02-08T06:40:14.137ZA Difference in extent metadata is expected here
27902024-02-08T06:40:14.137ZOn loop 12
27912024-02-08T06:40:14.137Z
27922024-02-08T06:40:14.140ZVerifying data now
27932024-02-08T06:40:19.645ZNo change after verify
27942024-02-08T06:40:19.665ZLoop: 12 Downstairs dump after verify (and repair):
27952024-02-08T06:40:19.665ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27962024-02-08T06:40:19.665Z 0 000-019 23 23 23 36 36 36 F F F
27972024-02-08T06:40:19.666Z 1 020-039 23 23 23 36 36 36 F F F
27982024-02-08T06:40:19.666Z 2 040-059 22 22 22 35 35 35 F F F
27992024-02-08T06:40:19.666Z 3 060-079 24 24 24 37 37 37 F F F
28002024-02-08T06:40:19.666Z 4 080-099 24 24 24 37 37 37 F F F
28012024-02-08T06:40:19.666Z 5 100-119 22 22 22 35 35 35 F F F
28022024-02-08T06:40:19.666Z 6 120-139 24 24 24 38 38 38 F F F
28032024-02-08T06:40:19.667Z 7 140-159 16 16 16 23 23 23 F F F
28042024-02-08T06:40:19.667Z 8 160-179 23 23 23 36 36 36 F F F
28052024-02-08T06:40:19.667Z 9 180-199 24 24 24 38 38 38 F F F
28062024-02-08T06:40:19.667Z 10 200-219 25 25 25 39 39 39 F F F
28072024-02-08T06:40:19.667Z 11 220-239 24 24 24 38 38 38 F F F
28082024-02-08T06:40:19.667Z 12 240-259 25 25 25 39 39 39 F F F
28092024-02-08T06:40:19.667Z 13 260-279 20 20 20 32 32 32 F F F
28102024-02-08T06:40:19.668Z 14 280-299 22 22 22 34 34 34 F F F
28112024-02-08T06:40:19.668Z 15 300-319 22 22 22 35 35 35 F F F
28122024-02-08T06:40:19.668Z 16 320-339 22 22 22 35 35 35 F F F
28132024-02-08T06:40:19.668Z 17 340-359 23 23 23 36 36 36 F F F
28142024-02-08T06:40:19.668Z 18 360-379 24 24 24 38 38 38 F F F
28152024-02-08T06:40:19.668Z 19 380-399 16 16 16 23 23 23 F F F
28162024-02-08T06:40:19.669Z 20 400-419 24 24 24 38 38 38 F F F
28172024-02-08T06:40:19.669Z 21 420-439 24 24 24 38 38 38 F F F
28182024-02-08T06:40:19.669Z 22 440-459 21 21 21 33 33 33 F F F
28192024-02-08T06:40:19.669Z 23 460-479 12 12 12 20 20 20 F F F
28202024-02-08T06:40:19.669Z 24 480-499 19 19 19 29 29 29 F F F
28212024-02-08T06:40:19.669Z 25 500-519 18 18 18 28 28 28 F F F
28222024-02-08T06:40:19.669Z 26 520-539 23 23 23 36 36 36 F F F
28232024-02-08T06:40:19.670Z 27 540-559 23 23 23 36 36 36 F F F
28242024-02-08T06:40:19.670Z 28 560-579 19 19 19 29 29 29 F F F
28252024-02-08T06:40:19.670Z 29 580-599 21 21 21 33 33 33 F F F
28262024-02-08T06:40:19.670ZMax gen: 25, Max flush: 39
28272024-02-08T06:40:19.713Z
28282024-02-08T06:40:19.713ZBegin loop 13
28292024-02-08T06:40:19.713ZDownstairs to restart: 1
28302024-02-08T06:40:19.713Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.922432217Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239}
28312024-02-08T06:40:19.714Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucibleCrucible runtime is spawned
28322024-02-08T06:40:19.714Z","level":30,"time":"2024-02-08T06:40:18.922736643Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239}
28332024-02-08T06:40:19.714Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.922769831Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239}
28342024-02-08T06:40:19.714Z{"msg":"Crucible 77589cb8-3213-44d6-bc51-44d4b17445ca has session id: 9ad9f930-c859-40e0-881b-2506512ad8fc","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.922805118Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28352024-02-08T06:40:19.714Z{"msg":"Upstairs opts: Upstairs UUID: 77589cb8-3213-44d6-bc51-44d4b17445ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.922835566Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28362024-02-08T06:40:19.714Z{"msg":"Crucible stats registered with UUID: 77589cb8-3213-44d6-bc51-44d4b17445ca","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.922864444Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28372024-02-08T06:40:19.714Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.922900741Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239}
28382024-02-08T06:40:19.714Z{"msg":"77589cb8-3213-44d6-bc51-44d4b17445ca active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.922926769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28392024-02-08T06:40:19.715Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.922960747Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28402024-02-08T06:40:19.715Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.923014813Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28412024-02-08T06:40:19.715Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.923044Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28422024-02-08T06:40:19.715Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.923097647Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28432024-02-08T06:40:19.715Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.923134374Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28442024-02-08T06:40:19.715Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.923153332Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28452024-02-08T06:40:19.715Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.92318247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28462024-02-08T06:40:19.716Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.923272343Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28472024-02-08T06:40:19.716Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.92332014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28482024-02-08T06:40:19.716Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.923373606Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28492024-02-08T06:40:19.716Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.923408003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28502024-02-08T06:40:19.716Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.923439471Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28512024-02-08T06:40:19.716Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.924137089Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28522024-02-08T06:40:19.716Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.924179785Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28532024-02-08T06:40:19.717Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.924207653Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28542024-02-08T06:40:19.717Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.924236371Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28552024-02-08T06:40:19.717Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.924804508Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28562024-02-08T06:40:19.717Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.924846915Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28572024-02-08T06:40:19.717Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.925201349Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28582024-02-08T06:40:19.717Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.925243045Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28592024-02-08T06:40:19.717Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.92544268Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28602024-02-08T06:40:19.717Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.925510375Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28612024-02-08T06:40:19.717Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.925555382Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28622024-02-08T06:40:19.718Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.92558523Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28632024-02-08T06:40:19.718Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.925608178Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28642024-02-08T06:40:19.718Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.925622497Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28652024-02-08T06:40:19.718Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.92584695Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28662024-02-08T06:40:19.718Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926302526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28672024-02-08T06:40:19.718Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926340623Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28682024-02-08T06:40:19.718Z{"msg":"[0]R flush_numbers[0..12]: [36, 36, 35, 37, 37, 35, 38, 23, 36, 38, 39, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926356902Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28692024-02-08T06:40:19.718Z{"msg":"[0]R generation[0..12]: [23, 23, 22, 24, 24, 22, 24, 16, 23, 24, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926371971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28702024-02-08T06:40:19.718Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926393439Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28712024-02-08T06:40:19.718Z{"msg":"[1]R flush_numbers[0..12]: [36, 36, 35, 37, 37, 35, 38, 23, 36, 38, 39, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926408068Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28722024-02-08T06:40:19.719Z{"msg":"[1]R generation[0..12]: [23, 23, 22, 24, 24, 22, 24, 16, 23, 24, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926428346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28732024-02-08T06:40:19.719Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926442765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28742024-02-08T06:40:19.719Z{"msg":"[2]R flush_numbers[0..12]: [36, 36, 35, 37, 37, 35, 38, 23, 36, 38, 39, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926456774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28752024-02-08T06:40:19.719Z{"msg":"[2]R generation[0..12]: [23, 23, 22, 24, 24, 22, 24, 16, 23, 24, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926477403Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28762024-02-08T06:40:19.719Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926491772Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28772024-02-08T06:40:19.719ZWait for a query_work_queue command to finish before sending IO
28782024-02-08T06:40:19.719Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.92651206Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28792024-02-08T06:40:19.720Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926525989Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28802024-02-08T06:40:19.720Z{"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926546128Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28812024-02-08T06:40:19.720Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926559816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28822024-02-08T06:40:19.720Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926573255Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28832024-02-08T06:40:19.720Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926591834Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28842024-02-08T06:40:19.720Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926605533Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28852024-02-08T06:40:19.720Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926625142Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28862024-02-08T06:40:19.720Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926638761Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28872024-02-08T06:40:19.720Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.9266531Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28882024-02-08T06:40:19.721Z{"msg":"77589cb8-3213-44d6-bc51-44d4b17445ca is now active with session: 9ad9f930-c859-40e0-881b-2506512ad8fc","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926673318Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28892024-02-08T06:40:19.721Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926685677Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239,"session_id":"9ad9f930-c859-40e0-881b-2506512ad8fc"}
28902024-02-08T06:40:19.721Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:18.926697966Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239}
28912024-02-08T06:40:19.721ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28922024-02-08T06:40:19.721ZLoading write count information from file "/tmp/test_repair_verify.data"
28932024-02-08T06:40:19.721ZRun Repair workload
28942024-02-08T06:40:19.721Z01/30 Write block 198 len 40960 data: 5 5 4 4 4 4 4 4 4 4
28952024-02-08T06:40:19.750Z02/30 Read block 392 len 4096
28962024-02-08T06:40:19.750Z03/30 Write block 73 len 8192 data: 3 3
28972024-02-08T06:40:19.751Z04/30 Write block 190 len 20480 data: 3 3 3 4 4
28982024-02-08T06:40:19.751Z05/30 Flush
28992024-02-08T06:40:19.751Z06/30 Write block 569 len 16384 data: 3 3 3 3
29002024-02-08T06:40:19.751Z07/30 Flush
29012024-02-08T06:40:19.751Z08/30 Write block 364 len 40960 data: 2 2 2 2 3 4 3 3 3 3
29022024-02-08T06:40:19.751Z09/30 Read block 218 len 12288
29032024-02-08T06:40:19.751Z10/30 Read block 308 len 4096
29042024-02-08T06:40:19.752Z11/30 Write block 533 len 8192 data: 3 3
29052024-02-08T06:40:19.752Z12/30 Flush
29062024-02-08T06:40:19.752Z13/30 Write block 48 len 40960 data: 4 3 4 4 4 4 5 5 6 6
29072024-02-08T06:40:19.752Z14/30 Read block 157 len 12288
29082024-02-08T06:40:19.752Z15/30 Write block 261 len 12288 data: 3 4 3
29092024-02-08T06:40:19.752Z16/30 Flush
29102024-02-08T06:40:19.752Z17/30 Write block 332 len 36864 data: 2 3 4 4 5 5 4 4 4
29112024-02-08T06:40:19.752Z18/30 Flush
29122024-02-08T06:40:19.752Z19/30 Write block 38 len 20480 data: 4 3 3 3 4
29132024-02-08T06:40:19.753Z20/30 Read block 229 len 40960
29142024-02-08T06:40:19.753Z21/30 Write block 20 len 8192 data: 2 2
29152024-02-08T06:40:19.753Z22/30 Write block 197 len 40960 data: 3 6 6 5 5 5 5 5 5 5
29162024-02-08T06:40:19.753Z23/30 Read block 69 len 4096
29172024-02-08T06:40:19.753Z24/30 Write block 589 len 24576 data: 3 2 2 2 2 2
29182024-02-08T06:40:19.753Z25/30 Write block 129 len 20480 data: 2 3 4 3 3
29192024-02-08T06:40:19.754Z26/30 Write block 162 len 32768 data: 5 5 6 5 3 3 3 3
29202024-02-08T06:40:19.754Z27/30 Read block 264 len 12288
29212024-02-08T06:40:19.754Z28/30 Write block 209 len 32768 data: 4 2 4 5 5 5 3 2
29222024-02-08T06:40:19.754Z29/30 Write block 260 len 20480 data: 3 4 5 4 3
29232024-02-08T06:40:19.754Z30/30 Write block 465 len 24576 data: 2 2 2 2 3 3
29242024-02-08T06:40:19.754Z----------------------------------------------------------------
29252024-02-08T06:40:19.754Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29262024-02-08T06:40:19.755ZStates: Active Active Active
29272024-02-08T06:40:19.755ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29282024-02-08T06:40:19.755Z 1 Acked 1000 Write 10 Done Done Done false
29292024-02-08T06:40:19.755Z 2 Acked 1001 Read 1 Done Done Done false
29302024-02-08T06:40:19.755Z 3 Acked 1002 Write 2 Done Done Done false
29312024-02-08T06:40:19.755Z 4 Acked 1003 Write 5 Done Sent Done false
29322024-02-08T06:40:19.755Z 5 Acked 1004 Flush 0 Done Sent Done false
29332024-02-08T06:40:19.755Z 6 Acked 1005 Write 4 Done Sent Done false
29342024-02-08T06:40:19.755Z 7 Acked 1006 Flush 0 Done Sent Done false
29352024-02-08T06:40:19.756Z 8 Acked 1007 Write 10 Done Sent Done false
29362024-02-08T06:40:19.756Z 9 Acked 1008 Read 3 Done Sent Done false
29372024-02-08T06:40:19.756Z 10 Acked 1009 Read 1 Done Sent Done false
29382024-02-08T06:40:19.756Z 11 Acked 1010 Write 2 Done Sent Done false
29392024-02-08T06:40:19.756Z 12 Acked 1011 Flush 0 Done Sent Done false
29402024-02-08T06:40:19.756Z 13 Acked 1012 Write 10 Done Sent Done false
29412024-02-08T06:40:19.756Z 14 Acked 1013 Read 3 Done Sent Done false
29422024-02-08T06:40:19.757Z 15 Acked 1014 Write 3 Done Sent Done false
29432024-02-08T06:40:19.757Z 16 Acked 1015 Flush 0 Done Sent Done false
29442024-02-08T06:40:19.757Z 17 Acked 1016 Write 9 Done Sent Done false
29452024-02-08T06:40:19.757Z 18 Acked 1017 Flush 0 Done Sent Done false
29462024-02-08T06:40:19.757Z 19 Acked 1018 Write 5 Done Sent Done false
29472024-02-08T06:40:19.757Z 20 Acked 1019 Read 10 Done Sent Done false
29482024-02-08T06:40:19.758Z 21 Acked 1020 Write 2 Done Sent Done false
29492024-02-08T06:40:19.758Z 22 Acked 1021 Write 10 Done Sent Done false
29502024-02-08T06:40:19.758Z 23 Acked 1022 Read 1 Done Sent Done false
29512024-02-08T06:40:19.758Z 24 Acked 1023 Write 6 Done Sent Done false
29522024-02-08T06:40:19.758Z 25 Acked 1024 Write 5 Done Sent Done false
29532024-02-08T06:40:19.758Z 26 Acked 1025 Write 8 Done Sent Done false
29542024-02-08T06:40:19.758Z 27 Acked 1026 Read 3 Done Sent Done false
29552024-02-08T06:40:19.758Z 28 Acked 1027 Write 8 Done Sent Done false
29562024-02-08T06:40:19.758Z 29 Acked 1028 Write 5 Done Sent Sent false
29572024-02-08T06:40:19.759Z 30 Acked 1029 Write 6 Sent Sent Sent false
29582024-02-08T06:40:19.759Z STATES DS:0 DS:1 DS:2 TOTAL
29592024-02-08T06:40:19.759Z New 0 0 0 0
29602024-02-08T06:40:19.759Z Sent 1 27 2 30
29612024-02-08T06:40:19.759Z Done 29 3 28 60
29622024-02-08T06:40:19.759Z Skipped 0 0 0 0
29632024-02-08T06:40:19.759Z Error 0 0 0 0
29642024-02-08T06:40:19.759ZLast Flush: 1017 0 1017
29652024-02-08T06:40:19.760ZDownstairs last five completed:
29662024-02-08T06:40:19.760ZUpstairs last five completed: 30 29 28 27 26
29672024-02-08T06:40:19.760Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:18.959519932Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1239}
29682024-02-08T06:40:19.760ZWrote out file "/tmp/test_repair_verify.data"
29692024-02-08T06:40:19.760Z
29702024-02-08T06:40:19.760ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29712024-02-08T06:40:19.779ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29722024-02-08T06:40:19.779Z 0 000-019 23 23 23 36 36 36 F F F
29732024-02-08T06:40:19.779Z 1 020-039 23 23 23 36 36 36 T F T <---
29742024-02-08T06:40:19.779Z 2 040-059 26 22 26 43 35 43 T F T <---
29752024-02-08T06:40:19.779Z 3 060-079 26 24 26 40 37 40 F T F <---
29762024-02-08T06:40:19.779Z 4 080-099 24 24 24 37 37 37 F F F
29772024-02-08T06:40:19.779Z 5 100-119 22 22 22 35 35 35 F F F
29782024-02-08T06:40:19.780Z 6 120-139 24 24 24 38 38 38 T F T <---
29792024-02-08T06:40:19.780Z 7 140-159 16 16 16 23 23 23 F F F
29802024-02-08T06:40:19.780Z 8 160-179 23 23 23 36 36 36 T F T <---
29812024-02-08T06:40:19.780Z 9 180-199 26 24 26 40 38 40 T T T <---
29822024-02-08T06:40:19.780Z 10 200-219 26 25 26 40 39 40 T T T <---
29832024-02-08T06:40:19.780Z 11 220-239 24 24 24 38 38 38 F F F
29842024-02-08T06:40:19.780Z 12 240-259 25 25 25 39 39 39 F F F
29852024-02-08T06:40:19.780Z 13 260-279 26 20 26 43 32 43 T F T <---
29862024-02-08T06:40:19.781Z 14 280-299 22 22 22 34 34 34 F F F
29872024-02-08T06:40:19.781Z 15 300-319 22 22 22 35 35 35 F F F
29882024-02-08T06:40:19.781Z 16 320-339 26 22 26 44 35 44 F F F <---
29892024-02-08T06:40:19.781Z 17 340-359 26 23 26 44 36 44 F F F <---
29902024-02-08T06:40:19.781Z 18 360-379 26 24 26 42 38 42 F F F <---
29912024-02-08T06:40:19.781Z 19 380-399 16 16 16 23 23 23 F F F
29922024-02-08T06:40:19.781Z 20 400-419 24 24 24 38 38 38 F F F
29932024-02-08T06:40:19.782Z 21 420-439 24 24 24 38 38 38 F F F
29942024-02-08T06:40:19.782Z 22 440-459 21 21 21 33 33 33 F F F
29952024-02-08T06:40:19.782Z 23 460-479 12 12 12 20 20 20 T F T <---
29962024-02-08T06:40:19.782Z 24 480-499 19 19 19 29 29 29 F F F
29972024-02-08T06:40:19.782Z 25 500-519 18 18 18 28 28 28 F F F
29982024-02-08T06:40:19.782Z 26 520-539 26 23 26 42 36 42 F F F <---
29992024-02-08T06:40:19.782Z 27 540-559 23 23 23 36 36 36 F F F
30002024-02-08T06:40:19.783Z 28 560-579 26 19 26 41 29 41 F F F <---
30012024-02-08T06:40:19.783Z 29 580-599 21 21 21 33 33 33 T F T <---
30022024-02-08T06:40:19.783ZMax gen: 26, Max flush: 44
30032024-02-08T06:40:19.783ZError: Difference in extent metadata found!
30042024-02-08T06:40:19.783ZA Difference in extent metadata is expected here
30052024-02-08T06:40:19.783ZOn loop 13
30062024-02-08T06:40:19.783Z
30072024-02-08T06:40:19.784ZVerifying data now
30082024-02-08T06:40:25.944ZNo change after verify
30092024-02-08T06:40:26.777ZLoop: 13 Downstairs dump after verify (and repair):
30102024-02-08T06:40:26.777ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30112024-02-08T06:40:26.777Z 0 000-019 23 23 23 36 36 36 F F F
30122024-02-08T06:40:26.777Z 1 020-039 27 27 27 45 45 45 F F F
30132024-02-08T06:40:26.777Z 2 040-059 27 27 27 45 45 45 F F F
30142024-02-08T06:40:26.777Z 3 060-079 26 26 26 40 40 40 F F F
30152024-02-08T06:40:26.777Z 4 080-099 24 24 24 37 37 37 F F F
30162024-02-08T06:40:26.777Z 5 100-119 22 22 22 35 35 35 F F F
30172024-02-08T06:40:26.777Z 6 120-139 27 27 27 45 45 45 F F F
30182024-02-08T06:40:26.777Z 7 140-159 16 16 16 23 23 23 F F F
30192024-02-08T06:40:26.777Z 8 160-179 27 27 27 45 45 45 F F F
30202024-02-08T06:40:26.778Z 9 180-199 27 27 27 45 45 45 F F F
30212024-02-08T06:40:26.778Z 10 200-219 27 27 27 45 45 45 F F F
30222024-02-08T06:40:26.778Z 11 220-239 24 24 24 38 38 38 F F F
30232024-02-08T06:40:26.778Z 12 240-259 25 25 25 39 39 39 F F F
30242024-02-08T06:40:26.778Z 13 260-279 27 27 27 45 45 45 F F F
30252024-02-08T06:40:26.778Z 14 280-299 22 22 22 34 34 34 F F F
30262024-02-08T06:40:26.778Z 15 300-319 22 22 22 35 35 35 F F F
30272024-02-08T06:40:26.778Z 16 320-339 26 26 26 44 44 44 F F F
30282024-02-08T06:40:26.778Z 17 340-359 26 26 26 44 44 44 F F F
30292024-02-08T06:40:26.778Z 18 360-379 26 26 26 42 42 42 F F F
30302024-02-08T06:40:26.778Z 19 380-399 16 16 16 23 23 23 F F F
30312024-02-08T06:40:26.778Z 20 400-419 24 24 24 38 38 38 F F F
30322024-02-08T06:40:26.778Z 21 420-439 24 24 24 38 38 38 F F F
30332024-02-08T06:40:26.778Z 22 440-459 21 21 21 33 33 33 F F F
30342024-02-08T06:40:26.778Z 23 460-479 27 27 27 45 45 45 F F F
30352024-02-08T06:40:26.779Z 24 480-499 19 19 19 29 29 29 F F F
30362024-02-08T06:40:26.779Z 25 500-519 18 18 18 28 28 28 F F F
30372024-02-08T06:40:26.779Z 26 520-539 26 26 26 42 42 42 F F F
30382024-02-08T06:40:26.779Z 27 540-559 23 23 23 36 36 36 F F F
30392024-02-08T06:40:26.779Z 28 560-579 26 26 26 41 41 41 F F F
30402024-02-08T06:40:26.779Z 29 580-599 27 27 27 45 45 45 F F F
30412024-02-08T06:40:26.779ZMax gen: 27, Max flush: 45
30422024-02-08T06:40:26.779Z
30432024-02-08T06:40:26.779ZBegin loop 14
30442024-02-08T06:40:26.779ZDownstairs to restart: 0
30452024-02-08T06:40:26.779Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.739285909Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247}
30462024-02-08T06:40:26.779Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\"Crucible runtime is spawned
30472024-02-08T06:40:26.779Z,\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.739605315Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247}
30482024-02-08T06:40:26.779Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.739650562Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247}
30492024-02-08T06:40:26.779Z{"msg":"Crucible 0ec571b9-179a-49e3-8c61-ba6a351a634f has session id: 5f50232f-d58b-4cef-bb6f-cba8521ecdfc","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.739671091Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30502024-02-08T06:40:26.780Z{"msg":"Upstairs opts: Upstairs UUID: 0ec571b9-179a-49e3-8c61-ba6a351a634f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.739704588Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30512024-02-08T06:40:26.780Z{"msg":"Crucible stats registered with UUID: 0ec571b9-179a-49e3-8c61-ba6a351a634f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.739721767Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30522024-02-08T06:40:26.780Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.739747635Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247}
30532024-02-08T06:40:26.780Z{"msg":"0ec571b9-179a-49e3-8c61-ba6a351a634f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.73981388Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30542024-02-08T06:40:26.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.739889534Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30552024-02-08T06:40:26.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.73994513Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30562024-02-08T06:40:26.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.739973458Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30572024-02-08T06:40:26.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740016834Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30582024-02-08T06:40:26.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740042043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30592024-02-08T06:40:26.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740105038Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30602024-02-08T06:40:26.780Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740147315Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30612024-02-08T06:40:26.780Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740186152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30622024-02-08T06:40:26.780Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740226389Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30632024-02-08T06:40:26.780Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740250157Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30642024-02-08T06:40:26.780Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740287224Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30652024-02-08T06:40:26.780Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740309232Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30662024-02-08T06:40:26.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740889689Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30672024-02-08T06:40:26.781Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740936985Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30682024-02-08T06:40:26.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.740989772Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30692024-02-08T06:40:26.781Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.741043277Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30702024-02-08T06:40:26.781Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.741580127Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30712024-02-08T06:40:26.781Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.741663371Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30722024-02-08T06:40:26.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.742052872Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30732024-02-08T06:40:26.781Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.742093059Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30742024-02-08T06:40:26.781Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.74220263Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30752024-02-08T06:40:26.781Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.742237138Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30762024-02-08T06:40:26.781Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.742253566Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30772024-02-08T06:40:26.781Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.742292754Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30782024-02-08T06:40:26.781Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.742308422Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30792024-02-08T06:40:26.781Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.742329601Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30802024-02-08T06:40:26.781Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.74261513Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30812024-02-08T06:40:26.781Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743074495Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30822024-02-08T06:40:26.781Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743113552Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30832024-02-08T06:40:26.782Z{"msg":"[0]R flush_numbers[0..12]: [36, 45, 45, 40, 37, 35, 45, 23, 45, 45, 45, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743131281Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30842024-02-08T06:40:26.782Z{"msg":"[0]R generation[0..12]: [23, 27, 27, 26, 24, 22, 27, 16, 27, 27, 27, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743146219Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30852024-02-08T06:40:26.782Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743168648Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30862024-02-08T06:40:26.782Z{"msg":"[1]R flush_numbers[0..12]: [36, 45, 45, 40, 37, 35, 45, 23, 45, 45, 45, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743183247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30872024-02-08T06:40:26.782Z{"msg":"[1]R generation[0..12]: [23, 27, 27, 26, 24, 22, 27, 16, 27, 27, 27, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743204505Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30882024-02-08T06:40:26.782Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743219794Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247Wait for a query_work_queue command to finish before sending IO
30892024-02-08T06:40:26.782Z,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30902024-02-08T06:40:26.782Z{"msg":"[2]R flush_numbers[0..12]: [36, 45, 45, 40, 37, 35, 45, 23, 45, 45, 45, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743246612Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30912024-02-08T06:40:26.782Z{"msg":"[2]R generation[0..12]: [23, 27, 27, 26, 24, 22, 27, 16, 27, 27, 27, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743261471Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30922024-02-08T06:40:26.782Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743283859Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30932024-02-08T06:40:26.782Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743299088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30942024-02-08T06:40:26.782Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743319677Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30952024-02-08T06:40:26.782Z{"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743332776Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30962024-02-08T06:40:26.782Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743352214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30972024-02-08T06:40:26.782Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743365853Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30982024-02-08T06:40:26.782Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743384382Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
30992024-02-08T06:40:26.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743397891Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
31002024-02-08T06:40:26.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743417559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
31012024-02-08T06:40:26.783Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743431138Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
31022024-02-08T06:40:26.783Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743450047Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
31032024-02-08T06:40:26.783Z{"msg":"0ec571b9-179a-49e3-8c61-ba6a351a634f is now active with session: 5f50232f-d58b-4cef-bb6f-cba8521ecdfc","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743463326Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
31042024-02-08T06:40:26.783Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743481644Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247,"session_id":"5f50232f-d58b-4cef-bb6f-cba8521ecdfc"}
31052024-02-08T06:40:26.783Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:24.743494524Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247}
31062024-02-08T06:40:26.783ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
31072024-02-08T06:40:26.783ZLoading write count information from file "/tmp/test_repair_verify.data"
31082024-02-08T06:40:26.783ZRun Repair workload
31092024-02-08T06:40:26.796Z01/30 Write block 177 len 24576 data: 3 4 4 3 4 4
31102024-02-08T06:40:26.796Z02/30 Write block 184 len 36864 data: 3 3 3 3 4 4 4 4 4
31112024-02-08T06:40:26.796Z03/30 Write block 220 len 24576 data: 3 4 5 5 6 6
31122024-02-08T06:40:26.796Z04/30 Read block 405 len 28672
31132024-02-08T06:40:26.797Z05/30 Read block 425 len 32768
31142024-02-08T06:40:26.797Z06/30 Read block 508 len 40960
31152024-02-08T06:40:26.797Z07/30 Write block 440 len 40960 data: 3 3 3 3 2 2 3 3 3 3
31162024-02-08T06:40:26.797Z08/30 Write block 565 len 24576 data: 3 3 3 3 4 4
31172024-02-08T06:40:26.797Z09/30 Read block 229 len 24576
31182024-02-08T06:40:26.797Z10/30 Write block 375 len 12288 data: 2 3 3
31192024-02-08T06:40:26.797Z11/30 Read block 495 len 16384
31202024-02-08T06:40:26.797Z12/30 Read block 241 len 28672
31212024-02-08T06:40:26.797Z13/30 Write block 556 len 4096 data: 5
31222024-02-08T06:40:26.797Z14/30 Write block 248 len 36864 data: 4 6 6 6 5 4 4 4 4
31232024-02-08T06:40:26.797Z15/30 Read block 461 len 24576
31242024-02-08T06:40:26.797Z16/30 Read block 406 len 12288
31252024-02-08T06:40:26.797Z17/30 Write block 510 len 4096 data: 5
31262024-02-08T06:40:26.797Z18/30 Read block 504 len 16384
31272024-02-08T06:40:26.797Z19/30 Read block 485 len 20480
31282024-02-08T06:40:26.798Z20/30 Flush
31292024-02-08T06:40:26.798Z21/30 Write block 89 len 20480 data: 4 6 7 8 7
31302024-02-08T06:40:26.798Z22/30 Read block 154 len 4096
31312024-02-08T06:40:26.798Z23/30 Write block 249 len 4096 data: 7
31322024-02-08T06:40:26.798Z24/30 Read block 404 len 32768
31332024-02-08T06:40:26.798Z25/30 Write block 400 len 36864 data: 3 3 3 3 3 3 3 2 2
31342024-02-08T06:40:26.798Z26/30 Read block 89 len 8192
31352024-02-08T06:40:26.798Z27/30 Read block 305 len 40960
31362024-02-08T06:40:26.798Z28/30 Write block 68 len 36864 data: 4 4 3 3 3 4 4 2 2
31372024-02-08T06:40:26.798Z29/30 Write block 472 len 4096 data: 4
31382024-02-08T06:40:26.798Z30/30 Write block 319 len 20480 data: 6 6 4 4 4
31392024-02-08T06:40:26.798Z----------------------------------------------------------------
31402024-02-08T06:40:26.798Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31412024-02-08T06:40:26.798ZStates: Active Active Active
31422024-02-08T06:40:26.798ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31432024-02-08T06:40:26.799Z 1 Acked 1000 Write 6 Sent Done Done false
31442024-02-08T06:40:26.799Z 2 Acked 1001 Write 9 Sent Done Done false
31452024-02-08T06:40:26.799Z 3 Acked 1002 Write 6 Sent Done Done false
31462024-02-08T06:40:26.799Z 4 Acked 1003 Read 7 Sent Done Done false
31472024-02-08T06:40:26.799Z 5 Acked 1004 Read 8 Sent Done Done false
31482024-02-08T06:40:26.799Z 6 Acked 1005 Read 10 Sent Done Done false
31492024-02-08T06:40:26.799Z 7 Acked 1006 Write 10 Sent Done Done false
31502024-02-08T06:40:26.799Z 8 Acked 1007 Write 6 Sent Done Done false
31512024-02-08T06:40:26.799Z 9 Acked 1008 Read 6 Sent Done Done false
31522024-02-08T06:40:26.799Z 10 Acked 1009 Write 3 Sent Done Done false
31532024-02-08T06:40:26.799Z 11 Acked 1010 Read 4 Sent Done Done false
31542024-02-08T06:40:26.799Z 12 Acked 1011 Read 7 Sent Done Done false
31552024-02-08T06:40:26.799Z 13 Acked 1012 Write 1 Sent Done Done false
31562024-02-08T06:40:26.799Z 14 Acked 1013 Write 9 Sent Done Done false
31572024-02-08T06:40:26.799Z 15 Acked 1014 Read 6 Sent Done Done false
31582024-02-08T06:40:26.799Z 16 Acked 1015 Read 3 Sent Done Done false
31592024-02-08T06:40:26.800Z 17 Acked 1016 Write 1 Sent Done Done false
31602024-02-08T06:40:26.800Z 18 Acked 1017 Read 4 Sent Done Done false
31612024-02-08T06:40:26.800Z 19 Acked 1018 Read 5 Sent Done Done false
31622024-02-08T06:40:26.800Z 20 Acked 1019 Flush 0 Sent Done Done false
31632024-02-08T06:40:26.800Z 21 Acked 1020 Write 5 Sent Done Done false
31642024-02-08T06:40:26.800Z 22 Acked 1021 Read 1 Sent Done Done false
31652024-02-08T06:40:26.800Z 23 Acked 1022 Write 1 Sent Done Done false
31662024-02-08T06:40:26.800Z 24 Acked 1023 Read 8 Sent Done Done false
31672024-02-08T06:40:26.800Z 25 Acked 1024 Write 9 Sent Done Done false
31682024-02-08T06:40:26.800Z 26 Acked 1025 Read 2 Sent Done Done false
31692024-02-08T06:40:26.800Z 27 Acked 1026 Read 10 Sent Done Done false
31702024-02-08T06:40:26.800Z 28 Acked 1027 Write 9 Sent Done Done false
31712024-02-08T06:40:26.800Z 29 Acked 1028 Write 1 Sent Done Done false
31722024-02-08T06:40:26.800Z 30 Acked 1029 Write 5 Sent Sent Sent false
31732024-02-08T06:40:26.800Z STATES DS:0 DS:1 DS:2 TOTAL
31742024-02-08T06:40:26.800Z New 0 0 0 0
31752024-02-08T06:40:26.800Z Sent 30 1 1 32
31762024-02-08T06:40:26.801Z Done 0 29 29 58
31772024-02-08T06:40:26.801Z Skipped 0 0 0 0
31782024-02-08T06:40:26.801Z Error 0 0 0 0
31792024-02-08T06:40:26.801ZLast Flush: 0 1019 1019
31802024-02-08T06:40:26.801ZDownstairs last five completed:
31812024-02-08T06:40:26.801ZUpstairs last five completed: 30 29 28 27 26
31822024-02-08T06:40:26.801Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:24.774524984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1247}
31832024-02-08T06:40:26.801ZWrote out file "/tmp/test_repair_verify.data"
31842024-02-08T06:40:26.801Z
31852024-02-08T06:40:26.801ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31862024-02-08T06:40:26.801ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31872024-02-08T06:40:26.801Z 0 000-019 23 23 23 36 36 36 F F F
31882024-02-08T06:40:26.801Z 1 020-039 27 27 27 45 45 45 F F F
31892024-02-08T06:40:26.801Z 2 040-059 27 27 27 45 45 45 F F F
31902024-02-08T06:40:26.801Z 3 060-079 26 26 26 40 40 40 F T T <---
31912024-02-08T06:40:26.801Z 4 080-099 24 24 24 37 37 37 F T T <---
31922024-02-08T06:40:26.801Z 5 100-119 22 22 22 35 35 35 F F F
31932024-02-08T06:40:26.801Z 6 120-139 27 27 27 45 45 45 F F F
31942024-02-08T06:40:26.801Z 7 140-159 16 16 16 23 23 23 F F F
31952024-02-08T06:40:26.802Z 8 160-179 27 28 28 45 46 46 F F F <---
31962024-02-08T06:40:26.802Z 9 180-199 27 28 28 45 46 46 F F F <---
31972024-02-08T06:40:26.802Z 10 200-219 27 27 27 45 45 45 F F F
31982024-02-08T06:40:26.802Z 11 220-239 24 28 28 38 46 46 F F F <---
31992024-02-08T06:40:26.802Z 12 240-259 25 28 28 39 46 46 F T T <---
32002024-02-08T06:40:26.802Z 13 260-279 27 27 27 45 45 45 F F F
32012024-02-08T06:40:26.802Z 14 280-299 22 22 22 34 34 34 F F F
32022024-02-08T06:40:26.802Z 15 300-319 22 22 22 35 35 35 F T T <---
32032024-02-08T06:40:26.802Z 16 320-339 26 26 26 44 44 44 F T T <---
32042024-02-08T06:40:26.802Z 17 340-359 26 26 26 44 44 44 F F F
32052024-02-08T06:40:26.802Z 18 360-379 26 28 28 42 46 46 F F F <---
32062024-02-08T06:40:26.802Z 19 380-399 16 16 16 23 23 23 F F F
32072024-02-08T06:40:26.802Z 20 400-419 24 24 24 38 38 38 F T T <---
32082024-02-08T06:40:26.802Z 21 420-439 24 24 24 38 38 38 F F F
32092024-02-08T06:40:26.809Z 22 440-459 21 28 28 33 46 46 F F F <---
32102024-02-08T06:40:26.810Z 23 460-479 27 27 27 45 45 45 F T T <---
32112024-02-08T06:40:26.810Z 24 480-499 19 19 19 29 29 29 F F F
32122024-02-08T06:40:26.810Z 25 500-519 18 28 28 28 46 46 F F F <---
32132024-02-08T06:40:26.810Z 26 520-539 26 26 26 42 42 42 F F F
32142024-02-08T06:40:26.810Z 27 540-559 23 28 28 36 46 46 F F F <---
32152024-02-08T06:40:26.810Z 28 560-579 26 28 28 41 46 46 F F F <---
32162024-02-08T06:40:26.810Z 29 580-599 27 27 27 45 45 45 F F F
32172024-02-08T06:40:26.810ZMax gen: 28, Max flush: 46
32182024-02-08T06:40:26.810ZError: Difference in extent metadata found!
32192024-02-08T06:40:26.810ZA Difference in extent metadata is expected here
32202024-02-08T06:40:26.810ZOn loop 14
32212024-02-08T06:40:26.810Z
32222024-02-08T06:40:26.810ZVerifying data now
32232024-02-08T06:40:30.264ZNo change after verify
32242024-02-08T06:40:30.264ZLoop: 14 Downstairs dump after verify (and repair):
32252024-02-08T06:40:30.279ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32262024-02-08T06:40:30.279Z 0 000-019 23 23 23 36 36 36 F F F
32272024-02-08T06:40:30.279Z 1 020-039 27 27 27 45 45 45 F F F
32282024-02-08T06:40:30.279Z 2 040-059 27 27 27 45 45 45 F F F
32292024-02-08T06:40:30.279Z 3 060-079 29 29 29 47 47 47 F F F
32302024-02-08T06:40:30.279Z 4 080-099 29 29 29 47 47 47 F F F
32312024-02-08T06:40:30.280Z 5 100-119 22 22 22 35 35 35 F F F
32322024-02-08T06:40:30.280Z 6 120-139 27 27 27 45 45 45 F F F
32332024-02-08T06:40:30.280Z 7 140-159 16 16 16 23 23 23 F F F
32342024-02-08T06:40:30.283Z 8 160-179 28 28 28 46 46 46 F F F
32352024-02-08T06:40:30.283Z 9 180-199 28 28 28 46 46 46 F F F
32362024-02-08T06:40:30.283Z 10 200-219 27 27 27 45 45 45 F F F
32372024-02-08T06:40:30.283Z 11 220-239 28 28 28 46 46 46 F F F
32382024-02-08T06:40:30.283Z 12 240-259 29 29 29 47 47 47 F F F
32392024-02-08T06:40:30.283Z 13 260-279 27 27 27 45 45 45 F F F
32402024-02-08T06:40:30.283Z 14 280-299 22 22 22 34 34 34 F F F
32412024-02-08T06:40:30.284Z 15 300-319 29 29 29 47 47 47 F F F
32422024-02-08T06:40:30.284Z 16 320-339 29 29 29 47 47 47 F F F
32432024-02-08T06:40:30.284Z 17 340-359 26 26 26 44 44 44 F F F
32442024-02-08T06:40:30.284Z 18 360-379 28 28 28 46 46 46 F F F
32452024-02-08T06:40:30.284Z 19 380-399 16 16 16 23 23 23 F F F
32462024-02-08T06:40:30.284Z 20 400-419 29 29 29 47 47 47 F F F
32472024-02-08T06:40:30.284Z 21 420-439 24 24 24 38 38 38 F F F
32482024-02-08T06:40:30.284Z 22 440-459 28 28 28 46 46 46 F F F
32492024-02-08T06:40:30.284Z 23 460-479 29 29 29 47 47 47 F F F
32502024-02-08T06:40:30.284Z 24 480-499 19 19 19 29 29 29 F F F
32512024-02-08T06:40:30.284Z 25 500-519 28 28 28 46 46 46 F F F
32522024-02-08T06:40:30.284Z 26 520-539 26 26 26 42 42 42 F F F
32532024-02-08T06:40:30.284Z 27 540-559 28 28 28 46 46 46 F F F
32542024-02-08T06:40:30.284Z 28 560-579 28 28 28 46 46 46 F F F
32552024-02-08T06:40:30.284Z 29 580-599 27 27 27 45 45 45 F F F
32562024-02-08T06:40:30.284ZMax gen: 29, Max flush: 47
32572024-02-08T06:40:30.285Z
32582024-02-08T06:40:30.285ZBegin loop 15
32592024-02-08T06:40:30.285ZDownstairs to restart: 0
32602024-02-08T06:40:30.299Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549144596Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255}
32612024-02-08T06:40:30.299Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549376959Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255}
32622024-02-08T06:40:30.299Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549402127Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255}
32632024-02-08T06:40:30.304Z{"msg":"Crucible 9ea5e991-b1d2-4d21-a978-b985b798bd82 has session id: 0b06232b-e3bf-45e4-9b75-9bb5f860898e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549434195Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32642024-02-08T06:40:30.305Z{"msg":"Upstairs opts: Upstairs UUID: 9ea5e991-b1d2-4d21-a978-b985b798bd82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549465312Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32652024-02-08T06:40:30.305ZCrucible runtime is spawned
32662024-02-08T06:40:30.305Z{"msg":"Crucible stats registered with UUID: 9ea5e991-b1d2-4d21-a978-b985b798bd82","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549521198Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32672024-02-08T06:40:30.305Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549560685Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255}
32682024-02-08T06:40:30.305Z{"msg":"9ea5e991-b1d2-4d21-a978-b985b798bd82 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549650298Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32692024-02-08T06:40:30.305Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549687966Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32702024-02-08T06:40:30.305Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549741202Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32712024-02-08T06:40:30.305Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.54976804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32722024-02-08T06:40:30.305Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549811996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32732024-02-08T06:40:30.305Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549835835Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32742024-02-08T06:40:30.305Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549873602Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32752024-02-08T06:40:30.305Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549900779Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32762024-02-08T06:40:30.305Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549947996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32772024-02-08T06:40:30.305Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.549988693Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32782024-02-08T06:40:30.305Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.550015631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32792024-02-08T06:40:30.306Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.550097895Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32802024-02-08T06:40:30.306Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.550153571Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32812024-02-08T06:40:30.306Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.550762245Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32822024-02-08T06:40:30.306Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.550805502Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32832024-02-08T06:40:30.306Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.550863647Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32842024-02-08T06:40:30.306Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.550903124Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32852024-02-08T06:40:30.306Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.551322983Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32862024-02-08T06:40:30.306Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.551442124Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32872024-02-08T06:40:30.306Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.551869611Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32882024-02-08T06:40:30.306Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.551912358Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32892024-02-08T06:40:30.306Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.551930407Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32902024-02-08T06:40:30.306Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.551944356Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32912024-02-08T06:40:30.306Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.551971494Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32922024-02-08T06:40:30.306Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.551985453Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32932024-02-08T06:40:30.306Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.552223415Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32942024-02-08T06:40:30.307Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.552257482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32952024-02-08T06:40:30.307Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.552708639Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32962024-02-08T06:40:30.312Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553145056Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32972024-02-08T06:40:30.312Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553182443Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32982024-02-08T06:40:30.312Z{"msg":"[0]R flush_numbers[0..12]: [36, 45, 45, 47, 47, 35, 45, 23, 46, 46, 45, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553198552Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
32992024-02-08T06:40:30.313Z{"msg":"[0]R generation[0..12]: [23, 27, 27, 29, 29, 22, 27, 16, 28, 28, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.55321972Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33002024-02-08T06:40:30.313Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553234429Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33012024-02-08T06:40:30.313Z{"msg":"[1]R flush_numbers[0..12]: [36, 45, 45, 47, 47, 35, 45, 23, 46, 46, 45, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553256427Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33022024-02-08T06:40:30.313Z{"msg":"[1]R generation[0..12]: [23, 27, 27, 29, 29, 22, 27, 16, 28, 28, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553270606Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33032024-02-08T06:40:30.313Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553291715Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33042024-02-08T06:40:30.313Z{"msg":"[2]R flush_numbers[0..12]: [36, 45, 45, 47, 47, 35, 45, 23, 46, 46, 45, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553306164Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO
33052024-02-08T06:40:30.313Z"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33062024-02-08T06:40:30.313Z{"msg":"[2]R generation[0..12]: [23, 27, 27, 29, 29, 22, 27, 16, 28, 28, 27, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553323893Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33072024-02-08T06:40:30.314Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553337861Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33082024-02-08T06:40:30.314Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.55335106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33092024-02-08T06:40:30.314Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553365099Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33102024-02-08T06:40:30.314Z{"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553378128Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33112024-02-08T06:40:30.314Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553398137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33122024-02-08T06:40:30.314Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553411846Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33132024-02-08T06:40:30.314Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553425195Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33142024-02-08T06:40:30.314Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553438654Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33152024-02-08T06:40:30.314Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553462002Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33162024-02-08T06:40:30.314Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553478311Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33172024-02-08T06:40:30.315Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.55349091Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33182024-02-08T06:40:30.315Z{"msg":"9ea5e991-b1d2-4d21-a978-b985b798bd82 is now active with session: 0b06232b-e3bf-45e4-9b75-9bb5f860898e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553503859Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33192024-02-08T06:40:30.315Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553516428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255,"session_id":"0b06232b-e3bf-45e4-9b75-9bb5f860898e"}
33202024-02-08T06:40:30.315Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:29.553536466Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255}
33212024-02-08T06:40:30.315ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33222024-02-08T06:40:30.315ZLoading write count information from file "/tmp/test_repair_verify.data"
33232024-02-08T06:40:30.315ZRun Repair workload
33242024-02-08T06:40:30.315Z01/30 Write block 298 len 8192 data: 5 7
33252024-02-08T06:40:30.315Z02/30 Read block 59 len 40960
33262024-02-08T06:40:30.316Z03/30 Read block 171 len 4096
33272024-02-08T06:40:30.316Z04/30 Read block 587 len 32768
33282024-02-08T06:40:30.316Z05/30 Write block 11 len 16384 data: 3 3 4 3
33292024-02-08T06:40:30.316Z06/30 Read block 141 len 40960
33302024-02-08T06:40:30.328Z07/30 Write block 299 len 32768 data: 8 6 5 5 4 4 4 3
33312024-02-08T06:40:30.328Z08/30 Read block 33 len 4096
33322024-02-08T06:40:30.329Z09/30 Write block 407 len 36864 data: 3 3 2 2 2 2 2 2 2
33332024-02-08T06:40:30.329Z10/30 Read block 357 len 24576
33342024-02-08T06:40:30.329Z11/30 Read block 375 len 28672
33352024-02-08T06:40:30.329Z12/30 Write block 455 len 40960 data: 5 5 5 5 4 3 3 4 4 2
33362024-02-08T06:40:30.329Z13/30 Read block 252 len 40960
33372024-02-08T06:40:30.330Z14/30 Read block 393 len 24576
33382024-02-08T06:40:30.330Z15/30 Flush
33392024-02-08T06:40:30.330Z16/30 Write block 382 len 16384 data: 3 3 5 4
33402024-02-08T06:40:30.353Z17/30 Flush
33412024-02-08T06:40:30.353Z18/30 Write block 522 len 20480 data: 2 2 2 2 2
33422024-02-08T06:40:30.354Z19/30 Read block 466 len 4096
33432024-02-08T06:40:30.354Z20/30 Read block 250 len 4096
33442024-02-08T06:40:30.354Z21/30 Write block 166 len 8192 data: 4 4
33452024-02-08T06:40:30.354Z22/30 Read block 488 len 20480
33462024-02-08T06:40:30.354Z23/30 Write block 537 len 20480 data: 4 3 3 3 3
33472024-02-08T06:40:30.354Z24/30 Write block 216 len 32768 data: 3 2 2 3 4 5 6 6
33482024-02-08T06:40:30.355Z25/30 Write block 195 len 12288 data: 2 2 4
33492024-02-08T06:40:30.355Z26/30 Flush
33502024-02-08T06:40:30.355Z27/30 Write block 403 len 40960 data: 4 4 4 4 4 4 3 3 3 3
33512024-02-08T06:40:30.355Z28/30 Write block 484 len 28672 data: 4 4 4 4 4 4 3
33522024-02-08T06:40:30.355Z29/30 Read block 462 len 12288
33532024-02-08T06:40:30.356Z30/30 Read block 225 len 40960
33542024-02-08T06:40:30.356Z----------------------------------------------------------------
33552024-02-08T06:40:30.356Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33562024-02-08T06:40:30.356ZStates: Active Active Active
33572024-02-08T06:40:30.357ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33582024-02-08T06:40:30.357Z 1 Acked 1000 Write 2 Done Done Done false
33592024-02-08T06:40:30.357Z 2 Acked 1001 Read 10 Sent Done Done false
33602024-02-08T06:40:30.357Z 3 Acked 1002 Read 1 Sent Done Done false
33612024-02-08T06:40:30.358Z 4 Acked 1003 Read 8 Sent Done Done false
33622024-02-08T06:40:30.358Z 5 Acked 1004 Write 4 Sent Done Done false
33632024-02-08T06:40:30.358Z 6 Acked 1005 Read 10 Sent Done Done false
33642024-02-08T06:40:30.358Z 7 Acked 1006 Write 8 Sent Done Done false
33652024-02-08T06:40:30.359Z 8 Acked 1007 Read 1 Sent Done Done false
33662024-02-08T06:40:30.359Z 9 Acked 1008 Write 9 Sent Done Done false
33672024-02-08T06:40:30.359Z 10 Acked 1009 Read 6 Sent Done Done false
33682024-02-08T06:40:30.359Z 11 Acked 1010 Read 7 Sent Done Done false
33692024-02-08T06:40:30.360Z 12 Acked 1011 Write 10 Sent Done Done false
33702024-02-08T06:40:30.360Z 13 Acked 1012 Read 10 Sent Done Done false
33712024-02-08T06:40:30.360Z 14 Acked 1013 Read 6 Sent Done Done false
33722024-02-08T06:40:30.360Z 15 Acked 1014 Flush 0 Sent Done Done false
33732024-02-08T06:40:30.360Z 16 Acked 1015 Write 4 Sent Done Done false
33742024-02-08T06:40:30.361Z 17 Acked 1016 Flush 0 Sent Done Done false
33752024-02-08T06:40:30.361Z 18 Acked 1017 Write 5 Sent Done Done false
33762024-02-08T06:40:30.361Z 19 Acked 1018 Read 1 Sent Done Done false
33772024-02-08T06:40:30.361Z 20 Acked 1019 Read 1 Sent Done Done false
33782024-02-08T06:40:30.361Z 21 Acked 1020 Write 2 Sent Done Done false
33792024-02-08T06:40:30.362Z 22 Acked 1021 Read 5 Sent Done Done false
33802024-02-08T06:40:30.362Z 23 Acked 1022 Write 5 Sent Done Done false
33812024-02-08T06:40:30.362Z 24 Acked 1023 Write 8 Sent Done Done false
33822024-02-08T06:40:30.362Z 25 Acked 1024 Write 3 Sent Done Done false
33832024-02-08T06:40:30.363Z 26 Acked 1025 Flush 0 Sent Done Done false
33842024-02-08T06:40:30.363Z 27 Acked 1026 Write 10 Sent Done Done false
33852024-02-08T06:40:30.363Z 28 Acked 1027 Write 7 Sent Done Done false
33862024-02-08T06:40:30.363Z 29 Acked 1028 Read 3 Sent Done Done false
33872024-02-08T06:40:30.363Z 30 Acked 1029 Read 10 Sent Done Done false
33882024-02-08T06:40:30.364Z STATES DS:0 DS:1 DS:2 TOTAL
33892024-02-08T06:40:30.364Z New 0 0 0 0
33902024-02-08T06:40:30.364Z Sent 29 0 0 29
33912024-02-08T06:40:30.364Z Done 1 30 30 61
33922024-02-08T06:40:30.364Z Skipped 0 0 0 0
33932024-02-08T06:40:30.365Z Error 0 0 0 0
33942024-02-08T06:40:30.365ZLast Flush: 0 1025 1025
33952024-02-08T06:40:30.365ZDownstairs last five completed:
33962024-02-08T06:40:30.365ZUpstairs last five completed: 30 29 28 27 26
33972024-02-08T06:40:30.365Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:29.585278993Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1255}
33982024-02-08T06:40:30.366ZWrote out file "/tmp/test_repair_verify.data"
33992024-02-08T06:40:30.366Z
34002024-02-08T06:40:30.377ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
34012024-02-08T06:40:30.380ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34022024-02-08T06:40:30.380Z 0 000-019 23 30 30 36 48 48 F F F <---
34032024-02-08T06:40:30.380Z 1 020-039 27 27 27 45 45 45 F F F
34042024-02-08T06:40:30.381Z 2 040-059 27 27 27 45 45 45 F F F
34052024-02-08T06:40:30.381Z 3 060-079 29 29 29 47 47 47 F F F
34062024-02-08T06:40:30.381Z 4 080-099 29 29 29 47 47 47 F F F
34072024-02-08T06:40:30.381Z 5 100-119 22 22 22 35 35 35 F F F
34082024-02-08T06:40:30.381Z 6 120-139 27 27 27 45 45 45 F F F
34092024-02-08T06:40:30.381Z 7 140-159 16 16 16 23 23 23 F F F
34102024-02-08T06:40:30.381Z 8 160-179 28 30 30 46 50 50 F F F <---
34112024-02-08T06:40:30.381Z 9 180-199 28 30 30 46 50 50 F F F <---
34122024-02-08T06:40:30.381Z 10 200-219 27 30 30 45 50 50 F F F <---
34132024-02-08T06:40:30.381Z 11 220-239 28 30 30 46 50 50 F F F <---
34142024-02-08T06:40:30.381Z 12 240-259 29 29 29 47 47 47 F F F
34152024-02-08T06:40:30.381Z 13 260-279 27 27 27 45 45 45 F F F
34162024-02-08T06:40:30.381Z 14 280-299 22 30 30 34 48 48 T F F <---
34172024-02-08T06:40:30.381Z 15 300-319 29 30 30 47 48 48 F F F <---
34182024-02-08T06:40:30.381Z 16 320-339 29 29 29 47 47 47 F F F
34192024-02-08T06:40:30.382Z 17 340-359 26 26 26 44 44 44 F F F
34202024-02-08T06:40:30.382Z 18 360-379 28 28 28 46 46 46 F F F
34212024-02-08T06:40:30.382Z 19 380-399 16 30 30 23 49 49 F F F <---
34222024-02-08T06:40:30.382Z 20 400-419 29 30 30 47 48 48 F T T <---
34232024-02-08T06:40:30.382Z 21 420-439 24 24 24 38 38 38 F F F
34242024-02-08T06:40:30.382Z 22 440-459 28 30 30 46 48 48 F F F <---
34252024-02-08T06:40:30.382Z 23 460-479 29 30 30 47 48 48 F F F <---
34262024-02-08T06:40:30.382Z 24 480-499 19 19 19 29 29 29 F T T <---
34272024-02-08T06:40:30.382Z 25 500-519 28 28 28 46 46 46 F F F
34282024-02-08T06:40:30.382Z 26 520-539 26 30 30 42 50 50 F F F <---
34292024-02-08T06:40:30.382Z 27 540-559 28 30 30 46 50 50 F F F <---
34302024-02-08T06:40:30.382Z 28 560-579 28 28 28 46 46 46 F F F
34312024-02-08T06:40:30.382Z 29 580-599 27 27 27 45 45 45 F F F
34322024-02-08T06:40:30.382ZMax gen: 30, Max flush: 50
34332024-02-08T06:40:30.382ZError: Difference in extent metadata found!
34342024-02-08T06:40:30.382ZA Difference in extent metadata is expected here
34352024-02-08T06:40:30.383ZOn loop 15
34362024-02-08T06:40:30.383Z
34372024-02-08T06:40:30.383ZVerifying data now
34382024-02-08T06:40:35.031ZNo change after verify
34392024-02-08T06:40:35.031ZLoop: 15 Downstairs dump after verify (and repair):
34402024-02-08T06:40:35.046ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34412024-02-08T06:40:35.046Z 0 000-019 30 30 30 48 48 48 F F F
34422024-02-08T06:40:35.046Z 1 020-039 27 27 27 45 45 45 F F F
34432024-02-08T06:40:35.046Z 2 040-059 27 27 27 45 45 45 F F F
34442024-02-08T06:40:35.046Z 3 060-079 29 29 29 47 47 47 F F F
34452024-02-08T06:40:35.046Z 4 080-099 29 29 29 47 47 47 F F F
34462024-02-08T06:40:35.046Z 5 100-119 22 22 22 35 35 35 F F F
34472024-02-08T06:40:35.047Z 6 120-139 27 27 27 45 45 45 F F F
34482024-02-08T06:40:35.047Z 7 140-159 16 16 16 23 23 23 F F F
34492024-02-08T06:40:35.047Z 8 160-179 30 30 30 50 50 50 F F F
34502024-02-08T06:40:35.047Z 9 180-199 30 30 30 50 50 50 F F F
34512024-02-08T06:40:35.050Z 10 200-219 30 30 30 50 50 50 F F F
34522024-02-08T06:40:35.050Z 11 220-239 30 30 30 50 50 50 F F F
34532024-02-08T06:40:35.050Z 12 240-259 29 29 29 47 47 47 F F F
34542024-02-08T06:40:35.050Z 13 260-279 27 27 27 45 45 45 F F F
34552024-02-08T06:40:35.050Z 14 280-299 30 30 30 48 48 48 F F F
34562024-02-08T06:40:35.050Z 15 300-319 30 30 30 48 48 48 F F F
34572024-02-08T06:40:35.050Z 16 320-339 29 29 29 47 47 47 F F F
34582024-02-08T06:40:35.050Z 17 340-359 26 26 26 44 44 44 F F F
34592024-02-08T06:40:35.050Z 18 360-379 28 28 28 46 46 46 F F F
34602024-02-08T06:40:35.050Z 19 380-399 30 30 30 49 49 49 F F F
34612024-02-08T06:40:35.050Z 20 400-419 31 31 31 51 51 51 F F F
34622024-02-08T06:40:35.051Z 21 420-439 24 24 24 38 38 38 F F F
34632024-02-08T06:40:35.051Z 22 440-459 30 30 30 48 48 48 F F F
34642024-02-08T06:40:35.051Z 23 460-479 30 30 30 48 48 48 F F F
34652024-02-08T06:40:35.051Z 24 480-499 31 31 31 51 51 51 F F F
34662024-02-08T06:40:35.051Z 25 500-519 28 28 28 46 46 46 F F F
34672024-02-08T06:40:35.051Z 26 520-539 30 30 30 50 50 50 F F F
34682024-02-08T06:40:35.051Z 27 540-559 30 30 30 50 50 50 F F F
34692024-02-08T06:40:35.051Z 28 560-579 28 28 28 46 46 46 F F F
34702024-02-08T06:40:35.051Z 29 580-599 27 27 27 45 45 45 F F F
34712024-02-08T06:40:35.051ZMax gen: 31, Max flush: 51
34722024-02-08T06:40:35.051Z
34732024-02-08T06:40:35.051ZBegin loop 16
34742024-02-08T06:40:35.051ZDownstairs to restart: 2
34752024-02-08T06:40:35.068Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.317838613Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263}
34762024-02-08T06:40:35.068ZCrucible runtime is spawned
34772024-02-08T06:40:35.073Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318187717Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263}
34782024-02-08T06:40:35.073Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318235463Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263}
34792024-02-08T06:40:35.073Z{"msg":"Crucible 4178d726-d62c-4f39-9673-cf453b205057 has session id: 568fe8b7-ebc1-4ede-878b-979909be2ad9","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318260721Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34802024-02-08T06:40:35.073Z{"msg":"Upstairs opts: Upstairs UUID: 4178d726-d62c-4f39-9673-cf453b205057, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318303208Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34812024-02-08T06:40:35.073Z{"msg":"Crucible stats registered with UUID: 4178d726-d62c-4f39-9673-cf453b205057","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318326267Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34822024-02-08T06:40:35.073Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318362034Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263}
34832024-02-08T06:40:35.073Z{"msg":"4178d726-d62c-4f39-9673-cf453b205057 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318377843Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34842024-02-08T06:40:35.073Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318394101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34852024-02-08T06:40:35.073Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318438448Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34862024-02-08T06:40:35.074Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318479625Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34872024-02-08T06:40:35.074Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318512642Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34882024-02-08T06:40:35.074Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318558839Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34892024-02-08T06:40:35.074Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318609005Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34902024-02-08T06:40:35.074Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.31867585Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34912024-02-08T06:40:35.074Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318707418Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34922024-02-08T06:40:35.074Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318743335Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34932024-02-08T06:40:35.074Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318769653Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34942024-02-08T06:40:35.074Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.318883465Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34952024-02-08T06:40:35.074Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:34.31895144Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34962024-02-08T06:40:35.074Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:34.319008325Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34972024-02-08T06:40:35.074Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:34.319086429Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34982024-02-08T06:40:35.074Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.319120467Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
34992024-02-08T06:40:35.074Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.319136416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35002024-02-08T06:40:35.074Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.319179052Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35012024-02-08T06:40:35.075Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.319517717Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35022024-02-08T06:40:35.075Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.319593411Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35032024-02-08T06:40:35.075Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.319628559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35042024-02-08T06:40:35.075Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.319645437Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35052024-02-08T06:40:35.075Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.32013658Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35062024-02-08T06:40:35.075Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.320162289Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35072024-02-08T06:40:35.075Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.320744525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35082024-02-08T06:40:35.075Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.320780192Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35092024-02-08T06:40:35.075Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.320795791Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35102024-02-08T06:40:35.075Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.32080931Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35112024-02-08T06:40:35.075Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.320830708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35122024-02-08T06:40:35.075Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:34.320845007Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35132024-02-08T06:40:36.071Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.320383404Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35142024-02-08T06:40:36.076Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.320586409Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35152024-02-08T06:40:36.076Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.321286976Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35162024-02-08T06:40:36.076Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.321320244Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35172024-02-08T06:40:36.076Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.321836095Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35182024-02-08T06:40:36.076Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322338937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35192024-02-08T06:40:36.076Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322393963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35202024-02-08T06:40:36.077Z{"msg":"[0]R flush_numbers[0..12]: [48, 45, 45, 47, 47, 35, 45, 23, 50, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322418811Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35212024-02-08T06:40:36.077Z{"msg":"[0]R generation[0..12]: [30, 27, 27, 29, 29, 22, 27, 16, 30, 30, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322447189Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35222024-02-08T06:40:36.077Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322466298Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35232024-02-08T06:40:36.077Z{"msg":"[1]R flush_numbers[0..12]: [48, 45, 45, 47, 47, 35, 45, 23, 50, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322484936Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35242024-02-08T06:40:36.077Z{"msg":"[1]R generation[0..12]: [30, 27, 27, 29, 29, 22, 27, 16, 30, 30, 30, 30]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
35252024-02-08T06:40:36.077Z,"time":"2024-02-08T06:40:35.322502165Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35262024-02-08T06:40:36.077Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322535733Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35272024-02-08T06:40:36.077Z{"msg":"[2]R flush_numbers[0..12]: [48, 45, 45, 47, 47, 35, 45, 23, 50, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322562191Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35282024-02-08T06:40:36.077Z{"msg":"[2]R generation[0..12]: [30, 27, 27, 29, 29, 22, 27, 16, 30, 30, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322579659Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35292024-02-08T06:40:36.078Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322609357Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35302024-02-08T06:40:36.078Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322622856Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35312024-02-08T06:40:36.078Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322636485Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35322024-02-08T06:40:36.078Z{"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322656853Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35332024-02-08T06:40:36.078Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322670323Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35342024-02-08T06:40:36.078Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322685591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35352024-02-08T06:40:36.078Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.32269778Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35362024-02-08T06:40:36.078Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322711289Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35372024-02-08T06:40:36.079Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322731788Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35382024-02-08T06:40:36.079Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322745637Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35392024-02-08T06:40:36.079Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322759216Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35402024-02-08T06:40:36.079Z{"msg":"4178d726-d62c-4f39-9673-cf453b205057 is now active with session: 568fe8b7-ebc1-4ede-878b-979909be2ad9","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322778704Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35412024-02-08T06:40:36.079Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322790873Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263,"session_id":"568fe8b7-ebc1-4ede-878b-979909be2ad9"}
35422024-02-08T06:40:36.079Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:35.322803772Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263}
35432024-02-08T06:40:36.079ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35442024-02-08T06:40:36.079ZLoading write count information from file "/tmp/test_repair_verify.data"
35452024-02-08T06:40:36.079ZRun Repair workload
35462024-02-08T06:40:36.079Z01/30 Write block 365 len 36864 data: 3 3 3 4 5 4 4 4 4
35472024-02-08T06:40:36.084Z02/30 Write block 243 len 20480 data: 5 4 4 4 5
35482024-02-08T06:40:36.084Z03/30 Write block 82 len 28672 data: 4 5 5 5 6 5 4
35492024-02-08T06:40:36.084Z04/30 Read block 92 len 12288
35502024-02-08T06:40:36.084Z05/30 Read block 489 len 8192
35512024-02-08T06:40:36.085Z06/30 Read block 489 len 36864
35522024-02-08T06:40:36.085Z07/30 Read block 258 len 32768
35532024-02-08T06:40:36.085Z08/30 Flush
35542024-02-08T06:40:36.085Z09/30 Write block 271 len 28672 data: 3 3 3 3 3 3 4
35552024-02-08T06:40:36.085Z10/30 Read block 336 len 12288
35562024-02-08T06:40:36.088Z11/30 Write block 434 len 12288 data: 3 3 4
35572024-02-08T06:40:36.088Z12/30 Read block 187 len 24576
35582024-02-08T06:40:36.089Z13/30 Read block 312 len 36864
35592024-02-08T06:40:36.089Z14/30 Flush
35602024-02-08T06:40:36.092Z15/30 Write block 373 len 4096 data: 5
35612024-02-08T06:40:36.092Z16/30 Write block 141 len 20480 data: 4 4 3 3 2
35622024-02-08T06:40:36.092Z17/30 Read block 64 len 28672
35632024-02-08T06:40:36.092Z18/30 Read block 187 len 4096
35642024-02-08T06:40:36.095Z19/30 Read block 468 len 4096
35652024-02-08T06:40:36.096Z20/30 Write block 182 len 4096 data: 5
35662024-02-08T06:40:36.096Z21/30 Write block 358 len 4096 data: 3
35672024-02-08T06:40:36.096Z22/30 Write block 184 len 16384 data: 4 4 4 4
35682024-02-08T06:40:36.096Z23/30 Write block 564 len 20480 data: 3 4 4 4 4
35692024-02-08T06:40:36.096Z24/30 Write block 482 len 28672 data: 3 4 5 5 5 5 5
35702024-02-08T06:40:36.096Z25/30 Read block 56 len 8192
35712024-02-08T06:40:36.099Z26/30 Read block 523 len 36864
35722024-02-08T06:40:36.099Z27/30 Read block 13 len 12288
35732024-02-08T06:40:36.100Z28/30 Read block 134 len 40960
35742024-02-08T06:40:36.100Z29/30 Read block 138 len 12288
35752024-02-08T06:40:36.104Z30/30 Read block 345 len 28672
35762024-02-08T06:40:36.104Z----------------------------------------------------------------
35772024-02-08T06:40:36.104Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35782024-02-08T06:40:36.104ZStates: Active Active Active
35792024-02-08T06:40:36.104ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35802024-02-08T06:40:36.104Z 1 Acked 1000 Write 9 Done Done Done false
35812024-02-08T06:40:36.104Z 2 Acked 1001 Write 5 Done Done Done false
35822024-02-08T06:40:36.104Z 3 Acked 1002 Write 7 Done Done Done false
35832024-02-08T06:40:36.105Z 4 Acked 1003 Read 3 Done Done Done false
35842024-02-08T06:40:36.105Z 5 Acked 1004 Read 2 Done Done Done false
35852024-02-08T06:40:36.105Z 6 Acked 1005 Read 9 Done Done Done false
35862024-02-08T06:40:36.105Z 7 Acked 1006 Read 8 Done Done Sent false
35872024-02-08T06:40:36.105Z 8 Acked 1007 Flush 0 Done Done Sent false
35882024-02-08T06:40:36.105Z 9 Acked 1008 Write 7 Done Done Sent false
35892024-02-08T06:40:36.105Z 10 Acked 1009 Read 3 Done Done Sent false
35902024-02-08T06:40:36.105Z 11 Acked 1010 Write 3 Done Done Sent false
35912024-02-08T06:40:36.106Z 12 Acked 1011 Read 6 Done Done Sent false
35922024-02-08T06:40:36.106Z 13 Acked 1012 Read 9 Done Done Sent false
35932024-02-08T06:40:36.106Z 14 Acked 1013 Flush 0 Done Done Sent false
35942024-02-08T06:40:36.106Z 15 Acked 1014 Write 1 Done Done Sent false
35952024-02-08T06:40:36.106Z 16 Acked 1015 Write 5 Done Done Sent false
35962024-02-08T06:40:36.106Z 17 Acked 1016 Read 7 Done Done Sent false
35972024-02-08T06:40:36.106Z 18 Acked 1017 Read 1 Done Done Sent false
35982024-02-08T06:40:36.106Z 19 Acked 1018 Read 1 Done Done Sent false
35992024-02-08T06:40:36.106Z 20 Acked 1019 Write 1 Done Done Sent false
36002024-02-08T06:40:36.107Z 21 Acked 1020 Write 1 Done Done Sent false
36012024-02-08T06:40:36.107Z 22 Acked 1021 Write 4 Done Done Sent false
36022024-02-08T06:40:36.107Z 23 Acked 1022 Write 5 Done Done Sent false
36032024-02-08T06:40:36.107Z 24 Acked 1023 Write 7 Done Done Sent false
36042024-02-08T06:40:36.107Z 25 Acked 1024 Read 2 Done Done Sent false
36052024-02-08T06:40:36.107Z 26 Acked 1025 Read 9 Done Done Sent false
36062024-02-08T06:40:36.107Z 27 Acked 1026 Read 3 Done Done Sent false
36072024-02-08T06:40:36.107Z 28 Acked 1027 Read 10 Done Done Sent false
36082024-02-08T06:40:36.107Z 29 Acked 1028 Read 3 Done Done Sent false
36092024-02-08T06:40:36.108Z 30 Acked 1029 Read 7 Done Done Sent false
36102024-02-08T06:40:36.108Z STATES DS:0 DS:1 DS:2 TOTAL
36112024-02-08T06:40:36.108Z New 0 0 0 0
36122024-02-08T06:40:36.108Z Sent 0 0 24 24
36132024-02-08T06:40:36.108Z Done 30 30 6 66
36142024-02-08T06:40:36.108Z Skipped 0 0 0 0
36152024-02-08T06:40:36.108Z Error 0 0 0 0
36162024-02-08T06:40:36.108ZLast Flush: 1013 1013 0
36172024-02-08T06:40:36.109ZDownstairs last five completed:
36182024-02-08T06:40:36.109ZUpstairs last five completed: 30 29 28 27 26
36192024-02-08T06:40:36.109Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:35.350504953Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1263}
36202024-02-08T06:40:36.109ZWrote out file "/tmp/test_repair_verify.data"
36212024-02-08T06:40:36.112Z
36222024-02-08T06:40:36.113ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
36232024-02-08T06:40:36.128ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36242024-02-08T06:40:36.128Z 0 000-019 30 30 30 48 48 48 F F F
36252024-02-08T06:40:36.128Z 1 020-039 27 27 27 45 45 45 F F F
36262024-02-08T06:40:36.128Z 2 040-059 27 27 27 45 45 45 F F F
36272024-02-08T06:40:36.128Z 3 060-079 29 29 29 47 47 47 F F F
36282024-02-08T06:40:36.128Z 4 080-099 32 32 29 52 52 47 F F T <---
36292024-02-08T06:40:36.128Z 5 100-119 22 22 22 35 35 35 F F F
36302024-02-08T06:40:36.128Z 6 120-139 27 27 27 45 45 45 F F F
36312024-02-08T06:40:36.129Z 7 140-159 16 16 16 23 23 23 T T F <---
36322024-02-08T06:40:36.129Z 8 160-179 30 30 30 50 50 50 F F F
36332024-02-08T06:40:36.133Z 9 180-199 30 30 30 50 50 50 T T F <---
36342024-02-08T06:40:36.133Z 10 200-219 30 30 30 50 50 50 F F F
36352024-02-08T06:40:36.133Z 11 220-239 30 30 30 50 50 50 F F F
36362024-02-08T06:40:36.133Z 12 240-259 32 32 29 52 52 47 F F T <---
36372024-02-08T06:40:36.133Z 13 260-279 32 32 27 53 53 45 F F F <---
36382024-02-08T06:40:36.134Z 14 280-299 30 30 30 48 48 48 F F F
36392024-02-08T06:40:36.134Z 15 300-319 30 30 30 48 48 48 F F F
36402024-02-08T06:40:36.134Z 16 320-339 29 29 29 47 47 47 F F F
36412024-02-08T06:40:36.134Z 17 340-359 26 26 26 44 44 44 T T F <---
36422024-02-08T06:40:36.134Z 18 360-379 32 32 28 52 52 46 T T T <---
36432024-02-08T06:40:36.134Z 19 380-399 30 30 30 49 49 49 F F F
36442024-02-08T06:40:36.134Z 20 400-419 31 31 31 51 51 51 F F F
36452024-02-08T06:40:36.134Z 21 420-439 32 32 24 53 53 38 F F F <---
36462024-02-08T06:40:36.134Z 22 440-459 30 30 30 48 48 48 F F F
36472024-02-08T06:40:36.134Z 23 460-479 30 30 30 48 48 48 F F F
36482024-02-08T06:40:36.135Z 24 480-499 31 31 31 51 51 51 T T F <---
36492024-02-08T06:40:36.135Z 25 500-519 28 28 28 46 46 46 F F F
36502024-02-08T06:40:36.135Z 26 520-539 30 30 30 50 50 50 F F F
36512024-02-08T06:40:36.135Z 27 540-559 30 30 30 50 50 50 F F F
36522024-02-08T06:40:36.135Z 28 560-579 28 28 28 46 46 46 T T F <---
36532024-02-08T06:40:36.135Z 29 580-599 27 27 27 45 45 45 F F F
36542024-02-08T06:40:36.135ZMax gen: 32, Max flush: 53
36552024-02-08T06:40:36.135ZError: Difference in extent metadata found!
36562024-02-08T06:40:36.135ZA Difference in extent metadata is expected here
36572024-02-08T06:40:36.135ZOn loop 16
36582024-02-08T06:40:36.135Z
36592024-02-08T06:40:36.149ZVerifying data now
36602024-02-08T06:40:41.646ZNo change after verify
36612024-02-08T06:40:41.652ZLoop: 16 Downstairs dump after verify (and repair):
36622024-02-08T06:40:41.660ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36632024-02-08T06:40:41.660Z 0 000-019 30 30 30 48 48 48 F F F
36642024-02-08T06:40:41.661Z 1 020-039 27 27 27 45 45 45 F F F
36652024-02-08T06:40:41.661Z 2 040-059 27 27 27 45 45 45 F F F
36662024-02-08T06:40:41.662Z 3 060-079 29 29 29 47 47 47 F F F
36672024-02-08T06:40:41.662Z 4 080-099 32 32 32 52 52 52 F F F
36682024-02-08T06:40:41.662Z 5 100-119 22 22 22 35 35 35 F F F
36692024-02-08T06:40:41.662Z 6 120-139 27 27 27 45 45 45 F F F
36702024-02-08T06:40:41.663Z 7 140-159 33 33 33 54 54 54 F F F
36712024-02-08T06:40:41.668Z 8 160-179 30 30 30 50 50 50 F F F
36722024-02-08T06:40:41.668Z 9 180-199 33 33 33 54 54 54 F F F
36732024-02-08T06:40:41.669Z 10 200-219 30 30 30 50 50 50 F F F
36742024-02-08T06:40:41.669Z 11 220-239 30 30 30 50 50 50 F F F
36752024-02-08T06:40:41.669Z 12 240-259 32 32 32 52 52 52 F F F
36762024-02-08T06:40:41.670Z 13 260-279 32 32 32 53 53 53 F F F
36772024-02-08T06:40:41.670Z 14 280-299 30 30 30 48 48 48 F F F
36782024-02-08T06:40:41.670Z 15 300-319 30 30 30 48 48 48 F F F
36792024-02-08T06:40:41.670Z 16 320-339 29 29 29 47 47 47 F F F
36802024-02-08T06:40:41.671Z 17 340-359 33 33 33 54 54 54 F F F
36812024-02-08T06:40:41.671Z 18 360-379 33 33 33 54 54 54 F F F
36822024-02-08T06:40:41.671Z 19 380-399 30 30 30 49 49 49 F F F
36832024-02-08T06:40:41.672Z 20 400-419 31 31 31 51 51 51 F F F
36842024-02-08T06:40:41.672Z 21 420-439 32 32 32 53 53 53 F F F
36852024-02-08T06:40:41.672Z 22 440-459 30 30 30 48 48 48 F F F
36862024-02-08T06:40:41.673Z 23 460-479 30 30 30 48 48 48 F F F
36872024-02-08T06:40:41.673Z 24 480-499 33 33 33 54 54 54 F F F
36882024-02-08T06:40:41.673Z 25 500-519 28 28 28 46 46 46 F F F
36892024-02-08T06:40:41.674Z 26 520-539 30 30 30 50 50 50 F F F
36902024-02-08T06:40:41.674Z 27 540-559 30 30 30 50 50 50 F F F
36912024-02-08T06:40:41.674Z 28 560-579 33 33 33 54 54 54 F F F
36922024-02-08T06:40:41.674Z 29 580-599 27 27 27 45 45 45 F F F
36932024-02-08T06:40:41.675ZMax gen: 33, Max flush: 54
36942024-02-08T06:40:41.675Z
36952024-02-08T06:40:41.676ZBegin loop 17
36962024-02-08T06:40:41.676ZDownstairs to restart: 1
36972024-02-08T06:40:41.682Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.92913572Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271}
36982024-02-08T06:40:41.682Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929394341Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271}
36992024-02-08T06:40:41.698Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
37002024-02-08T06:40:41.698Z,"time":"2024-02-08T06:40:40.929474075Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271}
37012024-02-08T06:40:41.699Z{"msg":"Crucible 56bf7b8c-6e0c-4657-a2f4-505ee3167eca has session id: 3f04cc9c-60b2-4343-ba5d-2bab70407782","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929519001Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37022024-02-08T06:40:41.699Z{"msg":"Upstairs opts: Upstairs UUID: 56bf7b8c-6e0c-4657-a2f4-505ee3167eca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929547699Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37032024-02-08T06:40:41.699Z{"msg":"Crucible stats registered with UUID: 56bf7b8c-6e0c-4657-a2f4-505ee3167eca","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929579427Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37042024-02-08T06:40:41.699Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929594706Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271}
37052024-02-08T06:40:41.700Z{"msg":"56bf7b8c-6e0c-4657-a2f4-505ee3167eca active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929642722Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37062024-02-08T06:40:41.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.92967584Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37072024-02-08T06:40:41.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929699008Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37082024-02-08T06:40:41.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929728816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37092024-02-08T06:40:41.701Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929743904Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37102024-02-08T06:40:41.701Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929763813Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37112024-02-08T06:40:41.701Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.92979965Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37122024-02-08T06:40:41.702Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929820698Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37132024-02-08T06:40:41.702Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929836727Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37142024-02-08T06:40:41.702Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929857486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37152024-02-08T06:40:41.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.92993396Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37162024-02-08T06:40:41.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.929984016Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37172024-02-08T06:40:41.702Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:40.930174902Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37182024-02-08T06:40:41.702Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:40.930221108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37192024-02-08T06:40:41.703Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:40.930275204Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37202024-02-08T06:40:41.703Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.930311892Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37212024-02-08T06:40:41.703Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.93034015Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37222024-02-08T06:40:41.703Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.930411414Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37232024-02-08T06:40:41.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.930809214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37242024-02-08T06:40:41.703Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.930886749Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37252024-02-08T06:40:41.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.930934655Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37262024-02-08T06:40:41.703Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.930960423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37272024-02-08T06:40:41.703Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.931469805Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37282024-02-08T06:40:41.703Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.931580717Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37292024-02-08T06:40:41.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.932003755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37302024-02-08T06:40:41.703Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.932041882Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37312024-02-08T06:40:41.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.932058421Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37322024-02-08T06:40:41.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.93207152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37332024-02-08T06:40:41.704Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.932086088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37342024-02-08T06:40:41.704Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:40.932107007Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37352024-02-08T06:40:42.689Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.931531922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37362024-02-08T06:40:42.689Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.931634804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37372024-02-08T06:40:42.714Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.932405606Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37382024-02-08T06:40:42.714Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.932450223Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37392024-02-08T06:40:42.714Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.932922178Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37402024-02-08T06:40:42.715Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933446108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37412024-02-08T06:40:42.715Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933479496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37422024-02-08T06:40:42.715Z{"msg":"[0]R flush_numbers[0..12]: [48, 45, 45, 47, 52, 35, 45, 54, 50, 54, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933495725Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37432024-02-08T06:40:42.715Z{"msg":"[0]R generation[0..12]: [30, 27, 27, 29, 32, 22, 27, 33, 30, 33, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.93356402Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37442024-02-08T06:40:42.715Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933589978Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37452024-02-08T06:40:42.715Z{"msg":"[1]R flush_numbers[0..12]: [48, 45, 45, 47, 52, 35, 45, 54, 50, 54, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933606896Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37462024-02-08T06:40:42.716Z{"msg":"[1]R generation[0..12]: [30, 27, 27, 29, 32, 22, 27, 33, 30, 33, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933628655Z","hostname":"Wait for a query_work_queue command to finish before sending IO
37472024-02-08T06:40:42.716Zip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37482024-02-08T06:40:42.716Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933647013Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37492024-02-08T06:40:42.716Z{"msg":"[2]R flush_numbers[0..12]: [48, 45, 45, 47, 52, 35, 45, 54, 50, 54, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933668822Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37502024-02-08T06:40:42.716Z{"msg":"[2]R generation[0..12]: [30, 27, 27, 29, 32, 22, 27, 33, 30, 33, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933683381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37512024-02-08T06:40:42.716Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933705799Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37522024-02-08T06:40:42.716Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933719928Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37532024-02-08T06:40:42.716Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933734547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37542024-02-08T06:40:42.717Z{"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933748136Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37552024-02-08T06:40:42.717Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933769014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37562024-02-08T06:40:42.717Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933783323Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37572024-02-08T06:40:42.717Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933802112Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37582024-02-08T06:40:42.717Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.93381716Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37592024-02-08T06:40:42.717Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933838819Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37602024-02-08T06:40:42.717Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933852658Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37612024-02-08T06:40:42.717Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933872547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37622024-02-08T06:40:42.718Z{"msg":"56bf7b8c-6e0c-4657-a2f4-505ee3167eca is now active with session: 3f04cc9c-60b2-4343-ba5d-2bab70407782","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933886525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37632024-02-08T06:40:42.718Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933906414Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271,"session_id":"3f04cc9c-60b2-4343-ba5d-2bab70407782"}
37642024-02-08T06:40:42.718Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:41.933919783Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271}
37652024-02-08T06:40:42.718ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37662024-02-08T06:40:42.718ZLoading write count information from file "/tmp/test_repair_verify.data"
37672024-02-08T06:40:42.718ZRun Repair workload
37682024-02-08T06:40:42.719Z01/30 Write block 10 len 16384 data: 4 4 4 5
37692024-02-08T06:40:42.719Z02/30 Write block 228 len 8192 data: 7 7
37702024-02-08T06:40:42.719Z03/30 Write block 145 len 20480 data: 3 2 2 2 4
37712024-02-08T06:40:42.719Z04/30 Write block 248 len 16384 data: 5 8 7 7
37722024-02-08T06:40:42.719Z05/30 Read block 92 len 40960
37732024-02-08T06:40:42.719Z06/30 Flush
37742024-02-08T06:40:42.719Z07/30 Write block 338 len 16384 data: 5 5 5 4
37752024-02-08T06:40:42.719Z08/30 Write block 1 len 40960 data: 2 2 3 3 3 3 4 4 4 5
37762024-02-08T06:40:42.719Z09/30 Write block 307 len 32768 data: 2 2 2 2 2 2 3 3
37772024-02-08T06:40:42.720Z10/30 Write block 22 len 16384 data: 3 3 4 5
37782024-02-08T06:40:42.720Z11/30 Read block 508 len 20480
37792024-02-08T06:40:42.720Z12/30 Read block 71 len 24576
37802024-02-08T06:40:42.720Z13/30 Read block 45 len 8192
37812024-02-08T06:40:42.720Z14/30 Write block 28 len 32768 data: 4 3 4 3 3 4 3 3
37822024-02-08T06:40:42.720Z15/30 Read block 593 len 24576
37832024-02-08T06:40:42.720Z16/30 Write block 382 len 4096 data: 4
37842024-02-08T06:40:42.720Z17/30 Write block 413 len 40960 data: 3 3 3 3 4 4 3 4 3 4
37852024-02-08T06:40:42.720Z18/30 Write block 173 len 16384 data: 4 2 3 3
37862024-02-08T06:40:42.720Z19/30 Write block 205 len 12288 data: 6 6 5
37872024-02-08T06:40:42.733Z20/30 Read block 279 len 16384
37882024-02-08T06:40:42.733Z21/30 Read block 47 len 20480
37892024-02-08T06:40:42.733Z22/30 Read block 425 len 28672
37902024-02-08T06:40:42.733Z23/30 Read block 580 len 20480
37912024-02-08T06:40:42.733Z24/30 Read block 132 len 28672
37922024-02-08T06:40:42.733Z25/30 Read block 88 len 40960
37932024-02-08T06:40:42.733Z26/30 Write block 549 len 16384 data: 5 3 4 4
37942024-02-08T06:40:42.733Z27/30 Read block 454 len 32768
37952024-02-08T06:40:42.734Z28/30 Read block 155 len 40960
37962024-02-08T06:40:42.734Z29/30 Read block 126 len 20480
37972024-02-08T06:40:42.734Z30/30 Read block 332 len 40960
37982024-02-08T06:40:42.734Z----------------------------------------------------------------
37992024-02-08T06:40:42.734Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
38002024-02-08T06:40:42.734ZStates: Active Active Active
38012024-02-08T06:40:42.734ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
38022024-02-08T06:40:42.734Z 1 Acked 1000 Write 4 Done Sent Done false
38032024-02-08T06:40:42.734Z 2 Acked 1001 Write 2 Done Sent Done false
38042024-02-08T06:40:42.734Z 3 Acked 1002 Write 5 Done Sent Done false
38052024-02-08T06:40:42.734Z 4 Acked 1003 Write 4 Done Sent Done false
38062024-02-08T06:40:42.735Z 5 Acked 1004 Read 10 Done Sent Done false
38072024-02-08T06:40:42.735Z 6 Acked 1005 Flush 0 Done Sent Done false
38082024-02-08T06:40:42.735Z 7 Acked 1006 Write 4 Done Sent Done false
38092024-02-08T06:40:42.735Z 8 Acked 1007 Write 10 Done Sent Done false
38102024-02-08T06:40:42.735Z 9 Acked 1008 Write 8 Done Sent Done false
38112024-02-08T06:40:42.736Z 10 Acked 1009 Write 4 Done Sent Done false
38122024-02-08T06:40:42.736Z 11 Acked 1010 Read 5 Done Sent Done false
38132024-02-08T06:40:42.736Z 12 Acked 1011 Read 6 Done Sent Done false
38142024-02-08T06:40:42.736Z 13 Acked 1012 Read 2 Done Sent Done false
38152024-02-08T06:40:42.736Z 14 Acked 1013 Write 8 Done Sent Done false
38162024-02-08T06:40:42.736Z 15 Acked 1014 Read 6 Done Sent Done false
38172024-02-08T06:40:42.736Z 16 Acked 1015 Write 1 Done Sent Done false
38182024-02-08T06:40:42.736Z 17 Acked 1016 Write 10 Done Sent Done false
38192024-02-08T06:40:42.736Z 18 Acked 1017 Write 4 Done Sent Done false
38202024-02-08T06:40:42.737Z 19 Acked 1018 Write 3 Done Sent Done false
38212024-02-08T06:40:42.737Z 20 Acked 1019 Read 4 Done Sent Done false
38222024-02-08T06:40:42.737Z 21 Acked 1020 Read 5 Done Sent Done false
38232024-02-08T06:40:42.737Z 22 Acked 1021 Read 7 Done Sent Done false
38242024-02-08T06:40:42.737Z 23 Acked 1022 Read 5 Done Sent Done false
38252024-02-08T06:40:42.737Z 24 Acked 1023 Read 7 Done Sent Done false
38262024-02-08T06:40:42.737Z 25 Acked 1024 Read 10 Done Sent Done false
38272024-02-08T06:40:42.737Z 26 Acked 1025 Write 4 Done Sent Done false
38282024-02-08T06:40:42.737Z 27 Acked 1026 Read 8 Done Sent Done false
38292024-02-08T06:40:42.737Z 28 Acked 1027 Read 10 Done Sent Done false
38302024-02-08T06:40:42.737Z 29 Acked 1028 Read 5 Done Sent Done false
38312024-02-08T06:40:42.737Z 30 Acked 1029 Read 10 Done Sent Done false
38322024-02-08T06:40:42.737Z STATES DS:0 DS:1 DS:2 TOTAL
38332024-02-08T06:40:42.737Z New 0 0 0 0
38342024-02-08T06:40:42.737Z Sent 0 30 0 30
38352024-02-08T06:40:42.737Z Done 30 0 30 60
38362024-02-08T06:40:42.738Z Skipped 0 0 0 0
38372024-02-08T06:40:42.738Z Error 0 0 0 0
38382024-02-08T06:40:42.738ZLast Flush: 1005 0 1005
38392024-02-08T06:40:42.738ZDownstairs last five completed:
38402024-02-08T06:40:42.738ZUpstairs last five completed: 30 29 28 27 26
38412024-02-08T06:40:42.738Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:41.9613989Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1271}
38422024-02-08T06:40:42.738ZWrote out file "/tmp/test_repair_verify.data"
38432024-02-08T06:40:42.738Z
38442024-02-08T06:40:42.738ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38452024-02-08T06:40:42.756ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38462024-02-08T06:40:42.756Z 0 000-019 34 30 34 55 48 55 T F T <---
38472024-02-08T06:40:42.756Z 1 020-039 27 27 27 45 45 45 T F T <---
38482024-02-08T06:40:42.756Z 2 040-059 27 27 27 45 45 45 F F F
38492024-02-08T06:40:42.756Z 3 060-079 29 29 29 47 47 47 F F F
38502024-02-08T06:40:42.757Z 4 080-099 32 32 32 52 52 52 F F F
38512024-02-08T06:40:42.757Z 5 100-119 22 22 22 35 35 35 F F F
38522024-02-08T06:40:42.757Z 6 120-139 27 27 27 45 45 45 F F F
38532024-02-08T06:40:42.758Z 7 140-159 34 33 34 55 54 55 F F F <---
38542024-02-08T06:40:42.758Z 8 160-179 30 30 30 50 50 50 T F T <---
38552024-02-08T06:40:42.758Z 9 180-199 33 33 33 54 54 54 F F F
38562024-02-08T06:40:42.758Z 10 200-219 30 30 30 50 50 50 T F T <---
38572024-02-08T06:40:42.759Z 11 220-239 34 30 34 55 50 55 F F F <---
38582024-02-08T06:40:42.759Z 12 240-259 34 32 34 55 52 55 F F F <---
38592024-02-08T06:40:42.759Z 13 260-279 32 32 32 53 53 53 F F F
38602024-02-08T06:40:42.760Z 14 280-299 30 30 30 48 48 48 F F F
38612024-02-08T06:40:42.760Z 15 300-319 30 30 30 48 48 48 T F T <---
38622024-02-08T06:40:42.760Z 16 320-339 29 29 29 47 47 47 T F T <---
38632024-02-08T06:40:42.761Z 17 340-359 33 33 33 54 54 54 T F T <---
38642024-02-08T06:40:42.761Z 18 360-379 33 33 33 54 54 54 F F F
38652024-02-08T06:40:42.762Z 19 380-399 30 30 30 49 49 49 T F T <---
38662024-02-08T06:40:42.762Z 20 400-419 31 31 31 51 51 51 T F T <---
38672024-02-08T06:40:42.762Z 21 420-439 32 32 32 53 53 53 T F T <---
38682024-02-08T06:40:42.762Z 22 440-459 30 30 30 48 48 48 F F F
38692024-02-08T06:40:42.763Z 23 460-479 30 30 30 48 48 48 F F F
38702024-02-08T06:40:42.763Z 24 480-499 33 33 33 54 54 54 F F F
38712024-02-08T06:40:42.763Z 25 500-519 28 28 28 46 46 46 F F F
38722024-02-08T06:40:42.764Z 26 520-539 30 30 30 50 50 50 F F F
38732024-02-08T06:40:42.764Z 27 540-559 30 30 30 50 50 50 T F T <---
38742024-02-08T06:40:42.764Z 28 560-579 33 33 33 54 54 54 F F F
38752024-02-08T06:40:42.764Z 29 580-599 27 27 27 45 45 45 F F F
38762024-02-08T06:40:42.765ZMax gen: 34, Max flush: 55
38772024-02-08T06:40:42.765ZError: Difference in extent metadata found!
38782024-02-08T06:40:42.765ZA Difference in extent metadata is expected here
38792024-02-08T06:40:42.766ZOn loop 17
38802024-02-08T06:40:42.766Z
38812024-02-08T06:40:42.772ZVerifying data now
38822024-02-08T06:40:47.432ZNo change after verify
38832024-02-08T06:40:47.433ZLoop: 17 Downstairs dump after verify (and repair):
38842024-02-08T06:40:47.452ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38852024-02-08T06:40:47.452Z 0 000-019 35 35 35 56 56 56 F F F
38862024-02-08T06:40:47.452Z 1 020-039 35 35 35 56 56 56 F F F
38872024-02-08T06:40:47.453Z 2 040-059 27 27 27 45 45 45 F F F
38882024-02-08T06:40:47.453Z 3 060-079 29 29 29 47 47 47 F F F
38892024-02-08T06:40:47.453Z 4 080-099 32 32 32 52 52 52 F F F
38902024-02-08T06:40:47.465Z 5 100-119 22 22 22 35 35 35 F F F
38912024-02-08T06:40:47.466Z 6 120-139 27 27 27 45 45 45 F F F
38922024-02-08T06:40:47.466Z 7 140-159 34 34 34 55 55 55 F F F
38932024-02-08T06:40:47.466Z 8 160-179 35 35 35 56 56 56 F F F
38942024-02-08T06:40:47.466Z 9 180-199 33 33 33 54 54 54 F F F
38952024-02-08T06:40:47.466Z 10 200-219 35 35 35 56 56 56 F F F
38962024-02-08T06:40:47.467Z 11 220-239 34 34 34 55 55 55 F F F
38972024-02-08T06:40:47.467Z 12 240-259 34 34 34 55 55 55 F F F
38982024-02-08T06:40:47.467Z 13 260-279 32 32 32 53 53 53 F F F
38992024-02-08T06:40:47.467Z 14 280-299 30 30 30 48 48 48 F F F
39002024-02-08T06:40:47.468Z 15 300-319 35 35 35 56 56 56 F F F
39012024-02-08T06:40:47.468Z 16 320-339 35 35 35 56 56 56 F F F
39022024-02-08T06:40:47.468Z 17 340-359 35 35 35 56 56 56 F F F
39032024-02-08T06:40:47.468Z 18 360-379 33 33 33 54 54 54 F F F
39042024-02-08T06:40:47.468Z 19 380-399 35 35 35 56 56 56 F F F
39052024-02-08T06:40:47.469Z 20 400-419 35 35 35 56 56 56 F F F
39062024-02-08T06:40:47.469Z 21 420-439 35 35 35 56 56 56 F F F
39072024-02-08T06:40:47.469Z 22 440-459 30 30 30 48 48 48 F F F
39082024-02-08T06:40:47.469Z 23 460-479 30 30 30 48 48 48 F F F
39092024-02-08T06:40:47.469Z 24 480-499 33 33 33 54 54 54 F F F
39102024-02-08T06:40:47.470Z 25 500-519 28 28 28 46 46 46 F F F
39112024-02-08T06:40:47.470Z 26 520-539 30 30 30 50 50 50 F F F
39122024-02-08T06:40:47.470Z 27 540-559 35 35 35 56 56 56 F F F
39132024-02-08T06:40:47.471Z 28 560-579 33 33 33 54 54 54 F F F
39142024-02-08T06:40:47.471Z 29 580-599 27 27 27 45 45 45 F F F
39152024-02-08T06:40:47.471ZMax gen: 35, Max flush: 56
39162024-02-08T06:40:47.472Z
39172024-02-08T06:40:47.472ZBegin loop 18
39182024-02-08T06:40:47.472ZDownstairs to restart: 1
39192024-02-08T06:40:47.488Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.710431003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279}
39202024-02-08T06:40:47.489Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.710744749Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279}
39212024-02-08T06:40:47.489Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.710788736Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279}
39222024-02-08T06:40:47.489Z{"msg":"Crucible 856f9d51-fe63-4f5c-a5eb-6810efc69ff1 has session id: 08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.710811734Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39232024-02-08T06:40:47.490ZCrucible runtime is spawned
39242024-02-08T06:40:47.490Z{"msg":"Upstairs opts: Upstairs UUID: 856f9d51-fe63-4f5c-a5eb-6810efc69ff1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.71087245Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39252024-02-08T06:40:47.490Z{"msg":"Crucible stats registered with UUID: 856f9d51-fe63-4f5c-a5eb-6810efc69ff1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.710915046Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39262024-02-08T06:40:47.491Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.710930415Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279}
39272024-02-08T06:40:47.491Z{"msg":"856f9d51-fe63-4f5c-a5eb-6810efc69ff1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.710955873Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39282024-02-08T06:40:47.491Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.710978552Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39292024-02-08T06:40:47.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.711014909Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39302024-02-08T06:40:47.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.711041427Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39312024-02-08T06:40:47.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.711064955Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39322024-02-08T06:40:47.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.711086234Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39332024-02-08T06:40:47.493Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.711116741Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39342024-02-08T06:40:47.493Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.711141329Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39352024-02-08T06:40:47.494Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.711226513Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39362024-02-08T06:40:47.494Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.71126433Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39372024-02-08T06:40:47.494Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.711417239Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39382024-02-08T06:40:47.495Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.711477804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39392024-02-08T06:40:47.495Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.711512741Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39402024-02-08T06:40:47.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.712232238Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39412024-02-08T06:40:47.496Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.712264285Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39422024-02-08T06:40:47.496Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.712279644Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39432024-02-08T06:40:47.496Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.712341339Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39442024-02-08T06:40:47.497Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.712922026Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39452024-02-08T06:40:47.497Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.713095023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39462024-02-08T06:40:47.497Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.713201315Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39472024-02-08T06:40:47.498Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.71325993Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39482024-02-08T06:40:47.498Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.713421518Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39492024-02-08T06:40:47.499Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.713465055Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39502024-02-08T06:40:47.499Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.713547499Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39512024-02-08T06:40:47.499Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.713587506Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39522024-02-08T06:40:47.500Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.713604524Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39532024-02-08T06:40:47.500Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.713625923Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39542024-02-08T06:40:47.500Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.713821958Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39552024-02-08T06:40:47.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714345599Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39562024-02-08T06:40:47.501Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714391145Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39572024-02-08T06:40:47.501Z{"msg":"[0]R flush_numbers[0..12]: [56, 56, 45, 47, 52, 35, 45, 55, 56, 54, 56, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714412854Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39582024-02-08T06:40:47.502Z{"msg":"[0]R generation[0..12]: [35, 35, 27, 29, 32, 22, 27, 34, 35, 33, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714440152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39592024-02-08T06:40:47.502Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.71445998Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39602024-02-08T06:40:47.502Z{"msg":"[1]R flush_numbers[0..12]: [56, 56, 45, 47, 52, 35, 45, 55, 56, 54, 56, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714493718Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"Wait for a query_work_queue command to finish before sending IO
39612024-02-08T06:40:47.503Z}
39622024-02-08T06:40:47.503Z{"msg":"[1]R generation[0..12]: [35, 35, 27, 29, 32, 22, 27, 34, 35, 33, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714514436Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39632024-02-08T06:40:47.503Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714541984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39642024-02-08T06:40:47.504Z{"msg":"[2]R flush_numbers[0..12]: [56, 56, 45, 47, 52, 35, 45, 55, 56, 54, 56, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714560113Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39652024-02-08T06:40:47.504Z{"msg":"[2]R generation[0..12]: [35, 35, 27, 29, 32, 22, 27, 34, 35, 33, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714578871Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39662024-02-08T06:40:47.504Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.71459739Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39672024-02-08T06:40:47.505Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714623098Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39682024-02-08T06:40:47.505Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714641227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39692024-02-08T06:40:47.505Z{"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714658225Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39702024-02-08T06:40:47.505Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714684394Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39712024-02-08T06:40:47.506Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714702152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39722024-02-08T06:40:47.506Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714719151Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39732024-02-08T06:40:47.506Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714736859Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39742024-02-08T06:40:47.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714762528Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39752024-02-08T06:40:47.507Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714780676Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39762024-02-08T06:40:47.507Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714797145Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39772024-02-08T06:40:47.507Z{"msg":"856f9d51-fe63-4f5c-a5eb-6810efc69ff1 is now active with session: 08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714822023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39782024-02-08T06:40:47.508Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714838272Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279,"session_id":"08e421c6-cef6-42ac-9bc4-9ad7ed5fa4d1"}
39792024-02-08T06:40:47.508Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:46.714855151Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279}
39802024-02-08T06:40:47.508ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39812024-02-08T06:40:47.509ZLoading write count information from file "/tmp/test_repair_verify.data"
39822024-02-08T06:40:47.509ZRun Repair workload
39832024-02-08T06:40:47.510Z01/30 Write block 516 len 32768 data: 2 3 4 4 4 2 3 3
39842024-02-08T06:40:47.510Z02/30 Read block 13 len 12288
39852024-02-08T06:40:47.510Z03/30 Write block 89 len 24576 data: 5 7 8 9 8 7
39862024-02-08T06:40:47.511Z04/30 Read block 16 len 8192
39872024-02-08T06:40:47.511Z05/30 Write block 486 len 40960 data: 6 6 6 5 4 3 3 2 3 3
39882024-02-08T06:40:47.511Z06/30 Write block 268 len 36864 data: 5 5 3 4 4 4 4 4 4
39892024-02-08T06:40:47.512Z07/30 Write block 36 len 4096 data: 3
39902024-02-08T06:40:47.512Z08/30 Write block 228 len 40960 data: 8 8 6 4 4 4 4 5 5 4
39912024-02-08T06:40:47.512Z09/30 Read block 80 len 12288
39922024-02-08T06:40:47.513Z10/30 Read block 495 len 4096
39932024-02-08T06:40:47.513Z11/30 Write block 239 len 32768 data: 4 3 4 4 6 5 5 5
39942024-02-08T06:40:47.513Z12/30 Read block 590 len 36864
39952024-02-08T06:40:47.547Z13/30 Write block 556 len 8192 data: 6 4
39962024-02-08T06:40:47.547Z14/30 Write block 68 len 36864 data: 5 5 4 4 4 5 5 3 3
39972024-02-08T06:40:47.547Z15/30 Read block 502 len 36864
39982024-02-08T06:40:47.548Z16/30 Read block 597 len 12288
39992024-02-08T06:40:47.548Z17/30 Read block 421 len 20480
40002024-02-08T06:40:47.548Z18/30 Write block 283 len 36864 data: 3 2 3 3 6 5 5 7 8
40012024-02-08T06:40:47.549Z19/30 Read block 302 len 12288
40022024-02-08T06:40:47.549Z20/30 Write block 18 len 12288 data: 3 3 3
40032024-02-08T06:40:47.549Z21/30 Write block 100 len 20480 data: 3 3 3 4 5
40042024-02-08T06:40:47.549Z22/30 Write block 103 len 24576 data: 5 6 4 5 5 5
40052024-02-08T06:40:47.550Z23/30 Write block 93 len 40960 data: 9 8 7 5 4 3 3 4 4 4
40062024-02-08T06:40:47.550Z24/30 Read block 102 len 8192
40072024-02-08T06:40:47.550Z25/30 Write block 22 len 4096 data: 4
40082024-02-08T06:40:47.551Z26/30 Read block 562 len 24576
40092024-02-08T06:40:47.551Z27/30 Write block 176 len 8192 data: 4 4
40102024-02-08T06:40:47.551Z28/30 Write block 3 len 4096 data: 4
40112024-02-08T06:40:47.552Z29/30 Write block 131 len 24576 data: 5 4 4 3 3 4
40122024-02-08T06:40:47.552Z30/30 Write block 194 len 40960 data: 5 3 3 5 7 7 6 6 6 6
40132024-02-08T06:40:47.552Z----------------------------------------------------------------
40142024-02-08T06:40:47.553Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
40152024-02-08T06:40:47.553ZStates: Active Active Active
40162024-02-08T06:40:47.553ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
40172024-02-08T06:40:47.554Z 1 Acked 1000 Write 8 Done Sent Done false
40182024-02-08T06:40:47.554Z 2 Acked 1001 Read 3 Done Sent Done false
40192024-02-08T06:40:47.555Z 3 Acked 1002 Write 6 Done Sent Done false
40202024-02-08T06:40:47.555Z 4 Acked 1003 Read 2 Done Sent Done false
40212024-02-08T06:40:47.555Z 5 Acked 1004 Write 10 Done Sent Done false
40222024-02-08T06:40:47.555Z 6 Acked 1005 Write 9 Done Sent Done false
40232024-02-08T06:40:47.556Z 7 Acked 1006 Write 1 Done Sent Done false
40242024-02-08T06:40:47.556Z 8 Acked 1007 Write 10 Done Sent Done false
40252024-02-08T06:40:47.556Z 9 Acked 1008 Read 3 Done Sent Done false
40262024-02-08T06:40:47.557Z 10 Acked 1009 Read 1 Done Sent Done false
40272024-02-08T06:40:47.557Z 11 Acked 1010 Write 8 Done Sent Done false
40282024-02-08T06:40:47.557Z 12 Acked 1011 Read 9 Done Sent Done false
40292024-02-08T06:40:47.557Z 13 Acked 1012 Write 2 Done Sent Done false
40302024-02-08T06:40:47.558Z 14 Acked 1013 Write 9 Done Sent Done false
40312024-02-08T06:40:47.558Z 15 Acked 1014 Read 9 Done Sent Done false
40322024-02-08T06:40:47.558Z 16 Acked 1015 Read 3 Done Sent Done false
40332024-02-08T06:40:47.558Z 17 Acked 1016 Read 5 Done Sent Done false
40342024-02-08T06:40:47.559Z 18 Acked 1017 Write 9 Done Sent Done false
40352024-02-08T06:40:47.559Z 19 Acked 1018 Read 3 Done Sent Done false
40362024-02-08T06:40:47.559Z 20 Acked 1019 Write 3 Done Sent Done false
40372024-02-08T06:40:47.559Z 21 Acked 1020 Write 5 Done Sent Done false
40382024-02-08T06:40:47.560Z 22 Acked 1021 Write 6 Done Sent Done false
40392024-02-08T06:40:47.560Z 23 Acked 1022 Write 10 Done Sent Done false
40402024-02-08T06:40:47.560Z 24 Acked 1023 Read 2 Done Sent Done false
40412024-02-08T06:40:47.560Z 25 Acked 1024 Write 1 Done Sent Done false
40422024-02-08T06:40:47.561Z 26 Acked 1025 Read 6 Done Sent Done false
40432024-02-08T06:40:47.561Z 27 Acked 1026 Write 2 Done Sent Done false
40442024-02-08T06:40:47.561Z 28 Acked 1027 Write 1 Done Sent Done false
40452024-02-08T06:40:47.561Z 29 Acked 1028 Write 6 Done Sent Done false
40462024-02-08T06:40:47.562Z 30 Acked 1029 Write 10 Sent Sent Sent false
40472024-02-08T06:40:47.562Z STATES DS:0 DS:1 DS:2 TOTAL
40482024-02-08T06:40:47.562Z New 0 0 0 0
40492024-02-08T06:40:47.563Z Sent 1 30 1 32
40502024-02-08T06:40:47.563Z Done 29 0 29 58
40512024-02-08T06:40:47.563Z Skipped 0 0 0 0
40522024-02-08T06:40:47.563Z Error 0 0 0 0
40532024-02-08T06:40:47.564ZLast Flush: 0 0 0
40542024-02-08T06:40:47.564ZDownstairs last five completed:
40552024-02-08T06:40:47.564ZUpstairs last five completed: 30 29 28 27 26
40562024-02-08T06:40:47.564Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:46.741282277Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1279}
40572024-02-08T06:40:47.565ZWrote out file "/tmp/test_repair_verify.data"
40582024-02-08T06:40:47.565Z
40592024-02-08T06:40:47.565ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40602024-02-08T06:40:47.579ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40612024-02-08T06:40:47.579Z 0 000-019 35 35 35 56 56 56 T F T <---
40622024-02-08T06:40:47.580Z 1 020-039 35 35 35 56 56 56 T F T <---
40632024-02-08T06:40:47.580Z 2 040-059 27 27 27 45 45 45 F F F
40642024-02-08T06:40:47.580Z 3 060-079 29 29 29 47 47 47 T F T <---
40652024-02-08T06:40:47.580Z 4 080-099 32 32 32 52 52 52 T F T <---
40662024-02-08T06:40:47.580Z 5 100-119 22 22 22 35 35 35 T F T <---
40672024-02-08T06:40:47.581Z 6 120-139 27 27 27 45 45 45 T F T <---
40682024-02-08T06:40:47.581Z 7 140-159 34 34 34 55 55 55 F F F
40692024-02-08T06:40:47.581Z 8 160-179 35 35 35 56 56 56 T F T <---
40702024-02-08T06:40:47.581Z 9 180-199 33 33 33 54 54 54 T F T <---
40712024-02-08T06:40:47.581Z 10 200-219 35 35 35 56 56 56 T F T <---
40722024-02-08T06:40:47.581Z 11 220-239 34 34 34 55 55 55 T F T <---
40732024-02-08T06:40:47.581Z 12 240-259 34 34 34 55 55 55 T F T <---
40742024-02-08T06:40:47.581Z 13 260-279 32 32 32 53 53 53 T F T <---
40752024-02-08T06:40:47.582Z 14 280-299 30 30 30 48 48 48 T F T <---
40762024-02-08T06:40:47.582Z 15 300-319 35 35 35 56 56 56 F F F
40772024-02-08T06:40:47.582Z 16 320-339 35 35 35 56 56 56 F F F
40782024-02-08T06:40:47.582Z 17 340-359 35 35 35 56 56 56 F F F
40792024-02-08T06:40:47.582Z 18 360-379 33 33 33 54 54 54 F F F
40802024-02-08T06:40:47.583Z 19 380-399 35 35 35 56 56 56 F F F
40812024-02-08T06:40:47.583Z 20 400-419 35 35 35 56 56 56 F F F
40822024-02-08T06:40:47.583Z 21 420-439 35 35 35 56 56 56 F F F
40832024-02-08T06:40:47.583Z 22 440-459 30 30 30 48 48 48 F F F
40842024-02-08T06:40:47.584Z 23 460-479 30 30 30 48 48 48 F F F
40852024-02-08T06:40:47.584Z 24 480-499 33 33 33 54 54 54 T F T <---
40862024-02-08T06:40:47.584Z 25 500-519 28 28 28 46 46 46 T F T <---
40872024-02-08T06:40:47.584Z 26 520-539 30 30 30 50 50 50 T F T <---
40882024-02-08T06:40:47.584Z 27 540-559 35 35 35 56 56 56 T F T <---
40892024-02-08T06:40:47.584Z 28 560-579 33 33 33 54 54 54 F F F
40902024-02-08T06:40:47.584Z 29 580-599 27 27 27 45 45 45 F F F
40912024-02-08T06:40:47.584ZMax gen: 35, Max flush: 56
40922024-02-08T06:40:47.585ZError: Difference in extent metadata found!
40932024-02-08T06:40:47.585ZA Difference in extent metadata is expected here
40942024-02-08T06:40:47.585ZOn loop 18
40952024-02-08T06:40:47.585Z
40962024-02-08T06:40:47.585ZVerifying data now
40972024-02-08T06:40:52.341ZNo change after verify
40982024-02-08T06:40:52.341ZLoop: 18 Downstairs dump after verify (and repair):
40992024-02-08T06:40:52.355ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
41002024-02-08T06:40:52.356Z 0 000-019 36 36 36 57 57 57 F F F
41012024-02-08T06:40:52.356Z 1 020-039 36 36 36 57 57 57 F F F
41022024-02-08T06:40:52.356Z 2 040-059 27 27 27 45 45 45 F F F
41032024-02-08T06:40:52.357Z 3 060-079 36 36 36 57 57 57 F F F
41042024-02-08T06:40:52.357Z 4 080-099 36 36 36 57 57 57 F F F
41052024-02-08T06:40:52.358Z 5 100-119 36 36 36 57 57 57 F F F
41062024-02-08T06:40:52.359Z 6 120-139 36 36 36 57 57 57 F F F
41072024-02-08T06:40:52.359Z 7 140-159 34 34 34 55 55 55 F F F
41082024-02-08T06:40:52.359Z 8 160-179 36 36 36 57 57 57 F F F
41092024-02-08T06:40:52.370Z 9 180-199 36 36 36 57 57 57 F F F
41102024-02-08T06:40:52.370Z 10 200-219 36 36 36 57 57 57 F F F
41112024-02-08T06:40:52.370Z 11 220-239 36 36 36 57 57 57 F F F
41122024-02-08T06:40:52.371Z 12 240-259 36 36 36 57 57 57 F F F
41132024-02-08T06:40:52.371Z 13 260-279 36 36 36 57 57 57 F F F
41142024-02-08T06:40:52.371Z 14 280-299 36 36 36 57 57 57 F F F
41152024-02-08T06:40:52.372Z 15 300-319 35 35 35 56 56 56 F F F
41162024-02-08T06:40:52.372Z 16 320-339 35 35 35 56 56 56 F F F
41172024-02-08T06:40:52.372Z 17 340-359 35 35 35 56 56 56 F F F
41182024-02-08T06:40:52.372Z 18 360-379 33 33 33 54 54 54 F F F
41192024-02-08T06:40:52.373Z 19 380-399 35 35 35 56 56 56 F F F
41202024-02-08T06:40:52.373Z 20 400-419 35 35 35 56 56 56 F F F
41212024-02-08T06:40:52.374Z 21 420-439 35 35 35 56 56 56 F F F
41222024-02-08T06:40:52.374Z 22 440-459 30 30 30 48 48 48 F F F
41232024-02-08T06:40:52.374Z 23 460-479 30 30 30 48 48 48 F F F
41242024-02-08T06:40:52.375Z 24 480-499 36 36 36 57 57 57 F F F
41252024-02-08T06:40:52.375Z 25 500-519 36 36 36 57 57 57 F F F
41262024-02-08T06:40:52.375Z 26 520-539 36 36 36 57 57 57 F F F
41272024-02-08T06:40:52.376Z 27 540-559 36 36 36 57 57 57 F F F
41282024-02-08T06:40:52.376Z 28 560-579 33 33 33 54 54 54 F F F
41292024-02-08T06:40:52.376Z 29 580-599 27 27 27 45 45 45 F F F
41302024-02-08T06:40:52.378ZMax gen: 36, Max flush: 57
41312024-02-08T06:40:52.378Z
41322024-02-08T06:40:52.378ZBegin loop 19
41332024-02-08T06:40:52.378ZDownstairs to restart: 1
41342024-02-08T06:40:52.393Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622098084Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287}
41352024-02-08T06:40:52.394Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
41362024-02-08T06:40:52.394Z,"time":"2024-02-08T06:40:51.622416771Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287}
41372024-02-08T06:40:52.394Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622449198Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287}
41382024-02-08T06:40:52.395Z{"msg":"Crucible 4c063050-9454-4cc1-8f19-15cb15835619 has session id: af9c0b75-6943-4e76-bd55-91007dc6eb39","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622475776Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41392024-02-08T06:40:52.395Z{"msg":"Upstairs opts: Upstairs UUID: 4c063050-9454-4cc1-8f19-15cb15835619, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622516883Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41402024-02-08T06:40:52.395Z{"msg":"Crucible stats registered with UUID: 4c063050-9454-4cc1-8f19-15cb15835619","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622541041Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41412024-02-08T06:40:52.396Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622572879Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287}
41422024-02-08T06:40:52.396Z{"msg":"4c063050-9454-4cc1-8f19-15cb15835619 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622588678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41432024-02-08T06:40:52.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622623165Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41442024-02-08T06:40:52.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622653193Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41452024-02-08T06:40:52.398Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622679881Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41462024-02-08T06:40:52.398Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622704749Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41472024-02-08T06:40:52.398Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622741256Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41482024-02-08T06:40:52.399Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622765234Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41492024-02-08T06:40:52.399Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622789743Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41502024-02-08T06:40:52.400Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622814081Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41512024-02-08T06:40:52.400Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622856858Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41522024-02-08T06:40:52.400Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:51.622914743Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41532024-02-08T06:40:52.400Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:51.622951551Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41542024-02-08T06:40:52.401Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.622979098Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41552024-02-08T06:40:52.401Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.623028325Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41562024-02-08T06:40:52.401Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:51.623058802Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41572024-02-08T06:40:52.401Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.62308997Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41582024-02-08T06:40:52.402Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.623112938Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41592024-02-08T06:40:52.402Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.623146616Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41602024-02-08T06:40:52.402Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.623667787Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41612024-02-08T06:40:52.403Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.623709863Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41622024-02-08T06:40:52.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.623725682Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41632024-02-08T06:40:52.403Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.62375671Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41642024-02-08T06:40:52.404Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.624356385Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41652024-02-08T06:40:52.404Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.624414571Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41662024-02-08T06:40:52.404Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.6249506Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41672024-02-08T06:40:52.405Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.624986738Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41682024-02-08T06:40:52.405Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.625023955Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41692024-02-08T06:40:52.405Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.625038244Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41702024-02-08T06:40:52.406Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.625052023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41712024-02-08T06:40:52.406Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:51.625073411Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41722024-02-08T06:40:53.380Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.624623237Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41732024-02-08T06:40:53.384Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.624914185Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41742024-02-08T06:40:53.384Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.625830306Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41752024-02-08T06:40:53.385Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.625874413Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41762024-02-08T06:40:53.385Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.626320269Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41772024-02-08T06:40:53.385Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.62685153Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41782024-02-08T06:40:53.385Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.626905236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41792024-02-08T06:40:53.385Z{"msg":"[0]R flush_numbers[0..12]: [57, 57, 45, 57, 57, 57, 57, 55, 57, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.626929294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41802024-02-08T06:40:53.385Z{"msg":"[0]R generation[0..12]: [36, 36, 27, 36, 36, 36, 36, 34, 36, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.626956442Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41812024-02-08T06:40:53.385Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.6269749Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41822024-02-08T06:40:53.385Z{"msg":"[1]R flush_numbers[0..12]: [57, 57, 45, 57, 57, 57, 57, 55, 57, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627004668Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41832024-02-08T06:40:53.385Z{"msg":"[1]R generation[0..12]: [36, 36, 27, 36, 36, 36, 36, 34, 36, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627041285Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
41842024-02-08T06:40:53.385Zaf9c0b75-6943-4e76-bd55-91007dc6eb39"}
41852024-02-08T06:40:53.385Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627064054Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41862024-02-08T06:40:53.385Z{"msg":"[2]R flush_numbers[0..12]: [57, 57, 45, 57, 57, 57, 57, 55, 57, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627090342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41872024-02-08T06:40:53.385Z{"msg":"[2]R generation[0..12]: [36, 36, 27, 36, 36, 36, 36, 34, 36, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.62710766Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41882024-02-08T06:40:53.386Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627134148Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41892024-02-08T06:40:53.386Z{"msg":"Max found gen is 37","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627151667Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41902024-02-08T06:40:53.386Z{"msg":"Generation requested: 38 >= found:37","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627176625Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41912024-02-08T06:40:53.386Z{"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627193634Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41922024-02-08T06:40:53.386Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627211063Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41932024-02-08T06:40:53.386Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627228651Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41942024-02-08T06:40:53.386Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.62724445Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41952024-02-08T06:40:53.386Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627262259Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41962024-02-08T06:40:53.386Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627279228Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41972024-02-08T06:40:53.386Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627304116Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41982024-02-08T06:40:53.386Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627319965Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
41992024-02-08T06:40:53.386Z{"msg":"4c063050-9454-4cc1-8f19-15cb15835619 is now active with session: af9c0b75-6943-4e76-bd55-91007dc6eb39","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627337213Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
42002024-02-08T06:40:53.386Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.627360431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287,"session_id":"af9c0b75-6943-4e76-bd55-91007dc6eb39"}
42012024-02-08T06:40:53.386Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:52.62737614Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287}
42022024-02-08T06:40:53.386ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
42032024-02-08T06:40:53.386ZLoading write count information from file "/tmp/test_repair_verify.data"
42042024-02-08T06:40:53.387ZRun Repair workload
42052024-02-08T06:40:53.387Z01/30 Write block 246 len 24576 data: 6 6 6 9 8 8
42062024-02-08T06:40:53.387Z02/30 Write block 418 len 36864 data: 5 4 5 4 5 4 4 3 2
42072024-02-08T06:40:53.387Z03/30 Write block 154 len 8192 data: 5 5
42082024-02-08T06:40:53.390Z04/30 Write block 528 len 24576 data: 2 2 3 3 3 4
42092024-02-08T06:40:53.390Z05/30 Write block 343 len 40960 data: 3 4 5 5 5 4 5 6 6 4
42102024-02-08T06:40:53.390Z06/30 Read block 514 len 32768
42112024-02-08T06:40:53.390Z07/30 Read block 117 len 12288
42122024-02-08T06:40:53.390Z08/30 Read block 134 len 12288
42132024-02-08T06:40:53.391Z09/30 Flush
42142024-02-08T06:40:53.394Z10/30 Write block 469 len 32768 data: 4 4 3 5 4 4 4 2
42152024-02-08T06:40:53.394Z11/30 Write block 519 len 4096 data: 5
42162024-02-08T06:40:53.394Z12/30 Read block 184 len 8192
42172024-02-08T06:40:53.394Z13/30 Write block 223 len 8192 data: 7 7
42182024-02-08T06:40:53.394Z14/30 Read block 486 len 28672
42192024-02-08T06:40:53.397Z15/30 Write block 397 len 36864 data: 3 3 3 4 4 4 5 5 5
42202024-02-08T06:40:53.397Z16/30 Write block 185 len 40960 data: 5 5 5 5 5 5 5 5 5 6
42212024-02-08T06:40:53.397Z17/30 Read block 77 len 16384
42222024-02-08T06:40:53.400Z18/30 Write block 325 len 40960 data: 3 2 2 2 2 2 2 3 4 5
42232024-02-08T06:40:53.400Z19/30 Read block 185 len 12288
42242024-02-08T06:40:53.400Z20/30 Read block 159 len 20480
42252024-02-08T06:40:53.403Z21/30 Write block 440 len 40960 data: 4 4 4 4 3 3 4 4 4 4
42262024-02-08T06:40:53.403Z22/30 Write block 423 len 32768 data: 5 5 4 3 2 2 2 3
42272024-02-08T06:40:53.403Z23/30 Read block 53 len 8192
42282024-02-08T06:40:53.406Z24/30 Read block 69 len 4096
42292024-02-08T06:40:53.406Z25/30 Write block 504 len 4096 data: 4
42302024-02-08T06:40:53.407Z26/30 Read block 565 len 12288
42312024-02-08T06:40:53.407Z27/30 Write block 521 len 40960 data: 3 4 4 3 3 3 2 3 3 4
42322024-02-08T06:40:53.407Z28/30 Write block 388 len 32768 data: 5 6 4 4 4 3 4 4
42332024-02-08T06:40:53.410Z29/30 Write block 372 len 40960 data: 5 6 2 3 4 4 3 3 3 3
42342024-02-08T06:40:53.410Z30/30 Write block 456 len 12288 data: 6 6 6
42352024-02-08T06:40:53.410Z----------------------------------------------------------------
42362024-02-08T06:40:53.410Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
42372024-02-08T06:40:53.410ZStates: Active Active Active
42382024-02-08T06:40:53.410ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
42392024-02-08T06:40:53.411Z 1 Acked 1000 Write 6 Done Done Done false
42402024-02-08T06:40:53.411Z 2 Acked 1001 Write 9 Done Sent Done false
42412024-02-08T06:40:53.411Z 3 Acked 1002 Write 2 Done Sent Done false
42422024-02-08T06:40:53.411Z 4 Acked 1003 Write 6 Done Sent Done false
42432024-02-08T06:40:53.411Z 5 Acked 1004 Write 10 Done Sent Done false
42442024-02-08T06:40:53.411Z 6 Acked 1005 Read 8 Done Sent Done false
42452024-02-08T06:40:53.411Z 7 Acked 1006 Read 3 Done Sent Done false
42462024-02-08T06:40:53.411Z 8 Acked 1007 Read 3 Done Sent Done false
42472024-02-08T06:40:53.411Z 9 Acked 1008 Flush 0 Done Sent Done false
42482024-02-08T06:40:53.411Z 10 Acked 1009 Write 8 Done Sent Done false
42492024-02-08T06:40:53.411Z 11 Acked 1010 Write 1 Done Sent Done false
42502024-02-08T06:40:53.411Z 12 Acked 1011 Read 2 Done Sent Done false
42512024-02-08T06:40:53.411Z 13 Acked 1012 Write 2 Done Sent Done false
42522024-02-08T06:40:53.411Z 14 Acked 1013 Read 7 Done Sent Done false
42532024-02-08T06:40:53.411Z 15 Acked 1014 Write 9 Done Sent Done false
42542024-02-08T06:40:53.412Z 16 Acked 1015 Write 10 Done Sent Done false
42552024-02-08T06:40:53.412Z 17 Acked 1016 Read 4 Done Sent Done false
42562024-02-08T06:40:53.412Z 18 Acked 1017 Write 10 Done Sent Done false
42572024-02-08T06:40:53.412Z 19 Acked 1018 Read 3 Done Sent Done false
42582024-02-08T06:40:53.412Z 20 Acked 1019 Read 5 Done Sent Done false
42592024-02-08T06:40:53.412Z 21 Acked 1020 Write 10 Done Sent Done false
42602024-02-08T06:40:53.412Z 22 Acked 1021 Write 8 Done Sent Done false
42612024-02-08T06:40:53.412Z 23 Acked 1022 Read 2 Done Sent Done false
42622024-02-08T06:40:53.412Z 24 Acked 1023 Read 1 Done Sent Done false
42632024-02-08T06:40:53.412Z 25 Acked 1024 Write 1 Done Sent Done false
42642024-02-08T06:40:53.412Z 26 Acked 1025 Read 3 Done Sent Done false
42652024-02-08T06:40:53.412Z 27 Acked 1026 Write 10 Done Sent Done false
42662024-02-08T06:40:53.412Z 28 Acked 1027 Write 8 Done Sent Done false
42672024-02-08T06:40:53.412Z 29 Acked 1028 Write 10 Sent Sent Sent false
42682024-02-08T06:40:53.412Z 30 Acked 1029 Write 3 Sent Sent Sent false
42692024-02-08T06:40:53.413Z STATES DS:0 DS:1 DS:2 TOTAL
42702024-02-08T06:40:53.413Z New 0 0 0 0
42712024-02-08T06:40:53.413Z Sent 2 29 2 33
42722024-02-08T06:40:53.413Z Done 28 1 28 57
42732024-02-08T06:40:53.413Z Skipped 0 0 0 0
42742024-02-08T06:40:53.413Z Error 0 0 0 0
42752024-02-08T06:40:53.413ZLast Flush: 1008 0 1008
42762024-02-08T06:40:53.413ZDownstairs last five completed:
42772024-02-08T06:40:53.413ZUpstairs last five completed: 30 29 28 27 26
42782024-02-08T06:40:53.413Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:52.656762884Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1287}
42792024-02-08T06:40:53.413ZWrote out file "/tmp/test_repair_verify.data"
42802024-02-08T06:40:53.416Z
42812024-02-08T06:40:53.419ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42822024-02-08T06:40:53.436ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42832024-02-08T06:40:53.436Z 0 000-019 36 36 36 57 57 57 F F F
42842024-02-08T06:40:53.436Z 1 020-039 36 36 36 57 57 57 F F F
42852024-02-08T06:40:53.436Z 2 040-059 27 27 27 45 45 45 F F F
42862024-02-08T06:40:53.436Z 3 060-079 36 36 36 57 57 57 F F F
42872024-02-08T06:40:53.436Z 4 080-099 36 36 36 57 57 57 F F F
42882024-02-08T06:40:53.436Z 5 100-119 36 36 36 57 57 57 F F F
42892024-02-08T06:40:53.436Z 6 120-139 36 36 36 57 57 57 F F F
42902024-02-08T06:40:53.437Z 7 140-159 38 34 38 58 55 58 F F F <---
42912024-02-08T06:40:53.437Z 8 160-179 36 36 36 57 57 57 F F F
42922024-02-08T06:40:53.437Z 9 180-199 36 36 36 57 57 57 T F T <---
42932024-02-08T06:40:53.443Z 10 200-219 36 36 36 57 57 57 F F F
42942024-02-08T06:40:53.443Z 11 220-239 36 36 36 57 57 57 T F T <---
42952024-02-08T06:40:53.443Z 12 240-259 38 36 38 58 57 58 F T F <---
42962024-02-08T06:40:53.443Z 13 260-279 36 36 36 57 57 57 F F F
42972024-02-08T06:40:53.443Z 14 280-299 36 36 36 57 57 57 F F F
42982024-02-08T06:40:53.443Z 15 300-319 35 35 35 56 56 56 F F F
42992024-02-08T06:40:53.443Z 16 320-339 35 35 35 56 56 56 T F T <---
43002024-02-08T06:40:53.443Z 17 340-359 38 35 38 58 56 58 F F F <---
43012024-02-08T06:40:53.443Z 18 360-379 33 33 33 54 54 54 T F T <---
43022024-02-08T06:40:53.443Z 19 380-399 35 35 35 56 56 56 T F T <---
43032024-02-08T06:40:53.443Z 20 400-419 38 35 38 58 56 58 T F T <---
43042024-02-08T06:40:53.444Z 21 420-439 38 35 38 58 56 58 T F T <---
43052024-02-08T06:40:53.444Z 22 440-459 30 30 30 48 48 48 T F T <---
43062024-02-08T06:40:53.444Z 23 460-479 30 30 30 48 48 48 T F T <---
43072024-02-08T06:40:53.444Z 24 480-499 36 36 36 57 57 57 F F F
43082024-02-08T06:40:53.444Z 25 500-519 36 36 36 57 57 57 T F T <---
43092024-02-08T06:40:53.444Z 26 520-539 38 36 38 58 57 58 T F T <---
43102024-02-08T06:40:53.444Z 27 540-559 36 36 36 57 57 57 F F F
43112024-02-08T06:40:53.444Z 28 560-579 33 33 33 54 54 54 F F F
43122024-02-08T06:40:53.444Z 29 580-599 27 27 27 45 45 45 F F F
43132024-02-08T06:40:53.444ZMax gen: 38, Max flush: 58
43142024-02-08T06:40:53.444ZError: Difference in extent metadata found!
43152024-02-08T06:40:53.444ZA Difference in extent metadata is expected here
43162024-02-08T06:40:53.445ZOn loop 19
43172024-02-08T06:40:53.445Z
43182024-02-08T06:40:53.449ZVerifying data now
43192024-02-08T06:40:59.128ZNo change after verify
43202024-02-08T06:40:59.128ZLoop: 19 Downstairs dump after verify (and repair):
43212024-02-08T06:40:59.142ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43222024-02-08T06:40:59.142Z 0 000-019 36 36 36 57 57 57 F F F
43232024-02-08T06:40:59.143Z 1 020-039 36 36 36 57 57 57 F F F
43242024-02-08T06:40:59.143Z 2 040-059 27 27 27 45 45 45 F F F
43252024-02-08T06:40:59.143Z 3 060-079 36 36 36 57 57 57 F F F
43262024-02-08T06:40:59.143Z 4 080-099 36 36 36 57 57 57 F F F
43272024-02-08T06:40:59.143Z 5 100-119 36 36 36 57 57 57 F F F
43282024-02-08T06:40:59.143Z 6 120-139 36 36 36 57 57 57 F F F
43292024-02-08T06:40:59.143Z 7 140-159 38 38 38 58 58 58 F F F
43302024-02-08T06:40:59.143Z 8 160-179 36 36 36 57 57 57 F F F
43312024-02-08T06:40:59.143Z 9 180-199 39 39 39 59 59 59 F F F
43322024-02-08T06:40:59.146Z 10 200-219 36 36 36 57 57 57 F F F
43332024-02-08T06:40:59.146Z 11 220-239 39 39 39 59 59 59 F F F
43342024-02-08T06:40:59.147Z 12 240-259 38 38 38 58 58 58 F F F
43352024-02-08T06:40:59.147Z 13 260-279 36 36 36 57 57 57 F F F
43362024-02-08T06:40:59.147Z 14 280-299 36 36 36 57 57 57 F F F
43372024-02-08T06:40:59.147Z 15 300-319 35 35 35 56 56 56 F F F
43382024-02-08T06:40:59.147Z 16 320-339 39 39 39 59 59 59 F F F
43392024-02-08T06:40:59.147Z 17 340-359 38 38 38 58 58 58 F F F
43402024-02-08T06:40:59.147Z 18 360-379 39 39 39 59 59 59 F F F
43412024-02-08T06:40:59.147Z 19 380-399 39 39 39 59 59 59 F F F
43422024-02-08T06:40:59.147Z 20 400-419 39 39 39 59 59 59 F F F
43432024-02-08T06:40:59.147Z 21 420-439 39 39 39 59 59 59 F F F
43442024-02-08T06:40:59.147Z 22 440-459 39 39 39 59 59 59 F F F
43452024-02-08T06:40:59.147Z 23 460-479 39 39 39 59 59 59 F F F
43462024-02-08T06:40:59.147Z 24 480-499 36 36 36 57 57 57 F F F
43472024-02-08T06:40:59.148Z 25 500-519 39 39 39 59 59 59 F F F
43482024-02-08T06:40:59.148Z 26 520-539 39 39 39 59 59 59 F F F
43492024-02-08T06:40:59.148Z 27 540-559 36 36 36 57 57 57 F F F
43502024-02-08T06:40:59.148Z 28 560-579 33 33 33 54 54 54 F F F
43512024-02-08T06:40:59.148Z 29 580-599 27 27 27 45 45 45 F F F
43522024-02-08T06:40:59.148ZMax gen: 39, Max flush: 59
43532024-02-08T06:40:59.148Z
43542024-02-08T06:40:59.148ZBegin loop 20
43552024-02-08T06:40:59.148ZDownstairs to restart: 2
43562024-02-08T06:40:59.164Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411041906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295}
43572024-02-08T06:40:59.164Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
43582024-02-08T06:40:59.164Z2024-02-08T06:40:58.411329244Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295}
43592024-02-08T06:40:59.168Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411375731Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295}
43602024-02-08T06:40:59.168Z{"msg":"Crucible a76ecbf5-ba64-43dc-86c7-090dfb6e75a8 has session id: a6fe0604-f54b-4814-a74b-d0d2953e1164","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411415098Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43612024-02-08T06:40:59.169Z{"msg":"Upstairs opts: Upstairs UUID: a76ecbf5-ba64-43dc-86c7-090dfb6e75a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411447995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43622024-02-08T06:40:59.169Z{"msg":"Crucible stats registered with UUID: a76ecbf5-ba64-43dc-86c7-090dfb6e75a8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411483903Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43632024-02-08T06:40:59.169Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411506991Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295}
43642024-02-08T06:40:59.169Z{"msg":"a76ecbf5-ba64-43dc-86c7-090dfb6e75a8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411580525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43652024-02-08T06:40:59.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411617313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43662024-02-08T06:40:59.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411663779Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43672024-02-08T06:40:59.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411708916Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43682024-02-08T06:40:59.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411735834Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43692024-02-08T06:40:59.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411766431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43702024-02-08T06:40:59.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.41179106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43712024-02-08T06:40:59.169Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411827957Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43722024-02-08T06:40:59.169Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411867894Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43732024-02-08T06:40:59.169Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411893622Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43742024-02-08T06:40:59.169Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411938678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43752024-02-08T06:40:59.169Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.411967026Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43762024-02-08T06:40:59.170Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:58.412078258Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43772024-02-08T06:40:59.170Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:58.412133564Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43782024-02-08T06:40:59.170Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:58.412286222Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43792024-02-08T06:40:59.170Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.41232106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43802024-02-08T06:40:59.170Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.412337619Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43812024-02-08T06:40:59.170Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.412439731Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43822024-02-08T06:40:59.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.412838531Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43832024-02-08T06:40:59.170Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.412873688Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43842024-02-08T06:40:59.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.412889487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43852024-02-08T06:40:59.170Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.412963312Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43862024-02-08T06:40:59.170Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.413447185Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43872024-02-08T06:40:59.170Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.413579105Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43882024-02-08T06:40:59.170Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.414020632Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43892024-02-08T06:40:59.170Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.414073208Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43902024-02-08T06:40:59.170Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.414119025Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43912024-02-08T06:40:59.171Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.414155612Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43922024-02-08T06:40:59.171Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.41418023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43932024-02-08T06:40:59.171Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:58.414194979Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43942024-02-08T06:41:00.167Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.413756504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43952024-02-08T06:41:00.167Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.413847797Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43962024-02-08T06:41:00.171Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.414681025Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43972024-02-08T06:41:00.172Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.414726021Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43982024-02-08T06:41:00.172Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.415202235Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
43992024-02-08T06:41:00.172Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.415748145Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44002024-02-08T06:41:00.172Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.415787062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44012024-02-08T06:41:00.172Z{"msg":"[0]R flush_numbers[0..12]: [57, 57, 45, 57, 57, 57, 57, 58, 57, 59, 57, 59]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.41580584Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44022024-02-08T06:41:00.172Z{"msg":"[0]R generation[0..12]: [36, 36, 27, 36, 36, 36, 36, 38, 36, 39, 36, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.415832808Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44032024-02-08T06:41:00.172Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.415851167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44042024-02-08T06:41:00.172Z{"msg":"[1]R flush_numbers[0..12]: [57, 57, 45, 57, 57, 57, 57, 58, 57, 59, 57, 59]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.415876815Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44052024-02-08T06:41:00.172Z{"msg":"[1]R generation[0..12]: [36, 36, 27, 36, 36, 36, 36, 38, 36, 39, 36, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.415894523Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44062024-02-08T06:41:00.172Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.415920192Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44072024-02-08T06:41:00.172Z{"msg":"[2]R flush_numbers[0..12]: [57, 57, 45, 57, 57, 57, 57, 58, 57, 59, 57, 59]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
44082024-02-08T06:41:00.172Z,"time":"2024-02-08T06:40:59.41593816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44092024-02-08T06:41:00.172Z{"msg":"[2]R generation[0..12]: [36, 36, 27, 36, 36, 36, 36, 38, 36, 39, 36, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.415966228Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44102024-02-08T06:41:00.172Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.415984757Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44112024-02-08T06:41:00.173Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416009605Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44122024-02-08T06:41:00.173Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416027973Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44132024-02-08T06:41:00.173Z{"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416053442Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44142024-02-08T06:41:00.173Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.41606851Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44152024-02-08T06:41:00.173Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416088679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44162024-02-08T06:41:00.173Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416101668Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44172024-02-08T06:41:00.173Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416116097Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44182024-02-08T06:41:00.173Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416135995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44192024-02-08T06:41:00.173Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416149724Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44202024-02-08T06:41:00.173Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416163513Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44212024-02-08T06:41:00.173Z{"msg":"a76ecbf5-ba64-43dc-86c7-090dfb6e75a8 is now active with session: a6fe0604-f54b-4814-a74b-d0d2953e1164","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416182992Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44222024-02-08T06:41:00.173Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.416195331Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295,"session_id":"a6fe0604-f54b-4814-a74b-d0d2953e1164"}
44232024-02-08T06:41:00.173Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:40:59.41620778Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295}
44242024-02-08T06:41:00.173ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
44252024-02-08T06:41:00.173ZLoading write count information from file "/tmp/test_repair_verify.data"
44262024-02-08T06:41:00.174ZRun Repair workload
44272024-02-08T06:41:00.174Z01/30 Write block 88 len 20480 data: 5 6 8 9 10
44282024-02-08T06:41:00.177Z02/30 Read block 467 len 16384
44292024-02-08T06:41:00.177Z03/30 Read block 32 len 28672
44302024-02-08T06:41:00.177Z04/30 Flush
44312024-02-08T06:41:00.177Z05/30 Write block 219 len 32768 data: 4 5 6 7 8 8 7 7
44322024-02-08T06:41:00.177Z06/30 Read block 552 len 28672
44332024-02-08T06:41:00.180Z07/30 Read block 387 len 32768
44342024-02-08T06:41:00.180Z08/30 Flush
44352024-02-08T06:41:00.183Z09/30 Write block 354 len 28672 data: 4 4 4 5 4 3 3
44362024-02-08T06:41:00.183Z10/30 Write block 191 len 40960 data: 6 6 6 7 4 4 6 8 8 7
44372024-02-08T06:41:00.184Z11/30 Write block 414 len 4096 data: 4
44382024-02-08T06:41:00.184Z12/30 Read block 493 len 12288
44392024-02-08T06:41:00.187Z13/30 Write block 14 len 32768 data: 4 3 2 3 4 4 4 3
44402024-02-08T06:41:00.187Z14/30 Read block 85 len 12288
44412024-02-08T06:41:00.187Z15/30 Read block 149 len 28672
44422024-02-08T06:41:00.190Z16/30 Read block 46 len 40960
44432024-02-08T06:41:00.190Z17/30 Flush
44442024-02-08T06:41:00.193Z18/30 Write block 67 len 12288 data: 3 6 6
44452024-02-08T06:41:00.193Z19/30 Read block 588 len 24576
44462024-02-08T06:41:00.196Z20/30 Flush
44472024-02-08T06:41:00.196Z21/30 Write block 257 len 28672 data: 3 3 3 4 5 6 5
44482024-02-08T06:41:00.199Z22/30 Write block 217 len 24576 data: 3 3 5 6 7 8
44492024-02-08T06:41:00.199Z23/30 Write block 267 len 32768 data: 4 6 6 4 5 5 5 5
44502024-02-08T06:41:00.199Z24/30 Write block 491 len 40960 data: 4 4 3 4 4 3 3 3 3 3
44512024-02-08T06:41:00.199Z25/30 Write block 438 len 4096 data: 3
44522024-02-08T06:41:00.199Z26/30 Write block 180 len 16384 data: 4 5 6 4
44532024-02-08T06:41:00.199Z27/30 Read block 321 len 20480
44542024-02-08T06:41:00.203Z28/30 Write block 587 len 32768 data: 2 3 4 3 3 3 3 3
44552024-02-08T06:41:00.203Z29/30 Write block 510 len 20480 data: 6 2 2 2 2
44562024-02-08T06:41:00.203Z30/30 Read block 528 len 32768
44572024-02-08T06:41:00.206Z----------------------------------------------------------------
44582024-02-08T06:41:00.206Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:10
44592024-02-08T06:41:00.207ZStates: Active Active Active
44602024-02-08T06:41:00.207ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44612024-02-08T06:41:00.207Z 21 Acked 1020 Write 7 Done Done Sent false
44622024-02-08T06:41:00.207Z 22 Acked 1021 Write 6 Done Done Sent false
44632024-02-08T06:41:00.207Z 23 Acked 1022 Write 8 Done Done Sent false
44642024-02-08T06:41:00.207Z 24 Acked 1023 Write 10 Done Done Sent false
44652024-02-08T06:41:00.207Z 25 Acked 1024 Write 1 Done Done Sent false
44662024-02-08T06:41:00.207Z 26 Acked 1025 Write 4 Done Done Sent false
44672024-02-08T06:41:00.207Z 27 Acked 1026 Read 5 Done Done Sent false
44682024-02-08T06:41:00.207Z 28 Acked 1027 Write 8 Done Done Sent false
44692024-02-08T06:41:00.207Z 29 Acked 1028 Write 5 Done Done Sent false
44702024-02-08T06:41:00.207Z 30 Acked 1029 Read 8 Done Sent Sent false
44712024-02-08T06:41:00.207Z STATES DS:0 DS:1 DS:2 TOTAL
44722024-02-08T06:41:00.207Z New 0 0 0 0
44732024-02-08T06:41:00.207Z Sent 0 1 10 11
44742024-02-08T06:41:00.208Z Done 10 9 0 19
44752024-02-08T06:41:00.208Z Skipped 0 0 0 0
44762024-02-08T06:41:00.208Z Error 0 0 0 0
44772024-02-08T06:41:00.208ZLast Flush: 1019 1019 1019
44782024-02-08T06:41:00.208ZDownstairs last five completed: 1019 1018 1017 1016 1015
44792024-02-08T06:41:00.208ZUpstairs last five completed: 30 29 28 27 26
44802024-02-08T06:41:00.208Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:40:59.450511724Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1295}
44812024-02-08T06:41:00.208ZWrote out file "/tmp/test_repair_verify.data"
44822024-02-08T06:41:00.214Z
44832024-02-08T06:41:00.217ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44842024-02-08T06:41:00.230ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44852024-02-08T06:41:00.230Z 0 000-019 40 40 40 62 62 62 F F F
44862024-02-08T06:41:00.230Z 1 020-039 40 40 40 62 62 62 F F F
44872024-02-08T06:41:00.230Z 2 040-059 27 27 27 45 45 45 F F F
44882024-02-08T06:41:00.230Z 3 060-079 40 40 40 63 63 63 F F F
44892024-02-08T06:41:00.230Z 4 080-099 40 40 40 60 60 60 F F F
44902024-02-08T06:41:00.231Z 5 100-119 36 36 36 57 57 57 F F F
44912024-02-08T06:41:00.231Z 6 120-139 36 36 36 57 57 57 F F F
44922024-02-08T06:41:00.234Z 7 140-159 38 38 38 58 58 58 F F F
44932024-02-08T06:41:00.234Z 8 160-179 36 36 36 57 57 57 F F F
44942024-02-08T06:41:00.234Z 9 180-199 40 40 40 62 62 62 T T F <---
44952024-02-08T06:41:00.234Z 10 200-219 40 40 40 62 62 62 T T F <---
44962024-02-08T06:41:00.234Z 11 220-239 40 40 40 61 61 61 T T F <---
44972024-02-08T06:41:00.234Z 12 240-259 38 38 38 58 58 58 T T F <---
44982024-02-08T06:41:00.234Z 13 260-279 36 36 36 57 57 57 T T F <---
44992024-02-08T06:41:00.234Z 14 280-299 36 36 36 57 57 57 F F F
45002024-02-08T06:41:00.234Z 15 300-319 35 35 35 56 56 56 F F F
45012024-02-08T06:41:00.235Z 16 320-339 39 39 39 59 59 59 F F F
45022024-02-08T06:41:00.235Z 17 340-359 40 40 40 62 62 62 F F F
45032024-02-08T06:41:00.235Z 18 360-379 40 40 40 62 62 62 F F F
45042024-02-08T06:41:00.235Z 19 380-399 39 39 39 59 59 59 F F F
45052024-02-08T06:41:00.235Z 20 400-419 40 40 40 62 62 62 F F F
45062024-02-08T06:41:00.235Z 21 420-439 39 39 39 59 59 59 T T F <---
45072024-02-08T06:41:00.235Z 22 440-459 39 39 39 59 59 59 F F F
45082024-02-08T06:41:00.235Z 23 460-479 39 39 39 59 59 59 F F F
45092024-02-08T06:41:00.235Z 24 480-499 36 36 36 57 57 57 T T F <---
45102024-02-08T06:41:00.235Z 25 500-519 39 39 39 59 59 59 T T F <---
45112024-02-08T06:41:00.235Z 26 520-539 39 39 39 59 59 59 F F F
45122024-02-08T06:41:00.235Z 27 540-559 36 36 36 57 57 57 F F F
45132024-02-08T06:41:00.235Z 28 560-579 33 33 33 54 54 54 F F F
45142024-02-08T06:41:00.235Z 29 580-599 27 27 27 45 45 45 T T F <---
45152024-02-08T06:41:00.235ZMax gen: 40, Max flush: 63
45162024-02-08T06:41:00.236ZError: Difference in extent metadata found!
45172024-02-08T06:41:00.236ZA Difference in extent metadata is expected here
45182024-02-08T06:41:00.236ZOn loop 20
45192024-02-08T06:41:00.236Z
45202024-02-08T06:41:00.239ZVerifying data now
45212024-02-08T06:41:04.730ZNo change after verify
45222024-02-08T06:41:04.730ZLoop: 20 Downstairs dump after verify (and repair):
45232024-02-08T06:41:04.745ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45242024-02-08T06:41:04.746Z 0 000-019 40 40 40 62 62 62 F F F
45252024-02-08T06:41:04.746Z 1 020-039 40 40 40 62 62 62 F F F
45262024-02-08T06:41:04.746Z 2 040-059 27 27 27 45 45 45 F F F
45272024-02-08T06:41:04.746Z 3 060-079 40 40 40 63 63 63 F F F
45282024-02-08T06:41:04.746Z 4 080-099 40 40 40 60 60 60 F F F
45292024-02-08T06:41:04.746Z 5 100-119 36 36 36 57 57 57 F F F
45302024-02-08T06:41:04.746Z 6 120-139 36 36 36 57 57 57 F F F
45312024-02-08T06:41:04.746Z 7 140-159 38 38 38 58 58 58 F F F
45322024-02-08T06:41:04.750Z 8 160-179 36 36 36 57 57 57 F F F
45332024-02-08T06:41:04.750Z 9 180-199 41 41 41 64 64 64 F F F
45342024-02-08T06:41:04.750Z 10 200-219 41 41 41 64 64 64 F F F
45352024-02-08T06:41:04.750Z 11 220-239 41 41 41 64 64 64 F F F
45362024-02-08T06:41:04.750Z 12 240-259 41 41 41 64 64 64 F F F
45372024-02-08T06:41:04.750Z 13 260-279 41 41 41 64 64 64 F F F
45382024-02-08T06:41:04.750Z 14 280-299 36 36 36 57 57 57 F F F
45392024-02-08T06:41:04.750Z 15 300-319 35 35 35 56 56 56 F F F
45402024-02-08T06:41:04.750Z 16 320-339 39 39 39 59 59 59 F F F
45412024-02-08T06:41:04.750Z 17 340-359 40 40 40 62 62 62 F F F
45422024-02-08T06:41:04.750Z 18 360-379 40 40 40 62 62 62 F F F
45432024-02-08T06:41:04.750Z 19 380-399 39 39 39 59 59 59 F F F
45442024-02-08T06:41:04.750Z 20 400-419 40 40 40 62 62 62 F F F
45452024-02-08T06:41:04.751Z 21 420-439 41 41 41 64 64 64 F F F
45462024-02-08T06:41:04.751Z 22 440-459 39 39 39 59 59 59 F F F
45472024-02-08T06:41:04.751Z 23 460-479 39 39 39 59 59 59 F F F
45482024-02-08T06:41:04.751Z 24 480-499 41 41 41 64 64 64 F F F
45492024-02-08T06:41:04.751Z 25 500-519 41 41 41 64 64 64 F F F
45502024-02-08T06:41:04.751Z 26 520-539 39 39 39 59 59 59 F F F
45512024-02-08T06:41:04.751Z 27 540-559 36 36 36 57 57 57 F F F
45522024-02-08T06:41:04.751Z 28 560-579 33 33 33 54 54 54 F F F
45532024-02-08T06:41:04.751Z 29 580-599 41 41 41 64 64 64 F F F
45542024-02-08T06:41:04.751ZMax gen: 41, Max flush: 64
45552024-02-08T06:41:04.751Z
45562024-02-08T06:41:04.751ZBegin loop 21
45572024-02-08T06:41:04.751ZDownstairs to restart: 2
45582024-02-08T06:41:04.766Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012219691Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303}
45592024-02-08T06:41:04.766ZCrucible runtime is spawned
45602024-02-08T06:41:04.772Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012529418Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303}
45612024-02-08T06:41:04.772Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012603642Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303}
45622024-02-08T06:41:04.773Z{"msg":"Crucible b9c4c1b3-e71d-4d55-8774-66e999c8743a has session id: 3182a6a4-cefd-449b-badc-c047fc3002a4","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012644809Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45632024-02-08T06:41:04.773Z{"msg":"Upstairs opts: Upstairs UUID: b9c4c1b3-e71d-4d55-8774-66e999c8743a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012684126Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45642024-02-08T06:41:04.773Z{"msg":"Crucible stats registered with UUID: b9c4c1b3-e71d-4d55-8774-66e999c8743a","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012723673Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45652024-02-08T06:41:04.773Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012743762Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303}
45662024-02-08T06:41:04.773Z{"msg":"b9c4c1b3-e71d-4d55-8774-66e999c8743a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012790368Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45672024-02-08T06:41:04.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012816526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45682024-02-08T06:41:04.773Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012843274Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45692024-02-08T06:41:04.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012910429Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45702024-02-08T06:41:04.773Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012936647Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45712024-02-08T06:41:04.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.012979474Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45722024-02-08T06:41:04.773Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013011242Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45732024-02-08T06:41:04.773Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013049449Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45742024-02-08T06:41:04.773Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013094865Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45752024-02-08T06:41:04.773Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013122023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45762024-02-08T06:41:04.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013159221Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45772024-02-08T06:41:04.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013185668Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45782024-02-08T06:41:04.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013218606Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45792024-02-08T06:41:04.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013816441Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45802024-02-08T06:41:04.774Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013853169Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45812024-02-08T06:41:04.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013868907Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45822024-02-08T06:41:04.774Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.013891575Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45832024-02-08T06:41:04.774Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.014481971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45842024-02-08T06:41:04.774Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.014531727Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45852024-02-08T06:41:04.774Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.015187558Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45862024-02-08T06:41:04.774Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.015224995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45872024-02-08T06:41:04.774Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.015249224Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45882024-02-08T06:41:04.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.015284351Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45892024-02-08T06:41:04.774Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.015306969Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45902024-02-08T06:41:04.774Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.015329468Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45912024-02-08T06:41:04.775Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.015344097Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45922024-02-08T06:41:04.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.015365055Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45932024-02-08T06:41:04.775Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.015777064Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45942024-02-08T06:41:04.780Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016218561Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45952024-02-08T06:41:04.780Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016255538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45962024-02-08T06:41:04.780Z{"msg":"[0]R flush_numbers[0..12]: [62, 62, 45, 63, 60, 57, 57, 58, 57, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016272717Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45972024-02-08T06:41:04.780Z{"msg":"[0]R generation[0..12]: [40, 40, 27, 40, 40, 36, 36, 38, 36, 41, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016295265Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45982024-02-08T06:41:04.780Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016310424Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
45992024-02-08T06:41:04.780Z{"msg":"[1]R flush_numbers[0..12]: [62, 62, 45, 63, 60, 57, 57, 58, 57, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016331532Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46002024-02-08T06:41:04.781Z{"msg":"[1]R generation[0..12]: [40, 40, 27, 40, 40, 36, 36, 38, 36, 41, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016345711Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46012024-02-08T06:41:04.781Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.01636519Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46022024-02-08T06:41:04.781Z{"msg":"[2]R flush_numbers[0..12]: [62, 62, 45, 63, 60, 57, 57, 58, 57, 64, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016398647Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46032024-02-08T06:41:04.781Z{"msg":"[2]R generation[0..12]: [40, 40, 27, 40, 40, 36, 36, 38, 36, 41, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016413356Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46042024-02-08T06:41:04.781Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016439854Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46052024-02-08T06:41:04.781Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016453953Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46062024-02-08T06:41:04.781ZWait for a query_work_queue command to finish before sending IO
46072024-02-08T06:41:04.781Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016470432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46082024-02-08T06:41:04.781Z{"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.01649074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46092024-02-08T06:41:04.781Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016504279Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46102024-02-08T06:41:04.781Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016528568Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46112024-02-08T06:41:04.781Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016544176Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46122024-02-08T06:41:04.781Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016558685Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46132024-02-08T06:41:04.781Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016592893Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46142024-02-08T06:41:04.782Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.01663283Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46152024-02-08T06:41:04.782Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016658628Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46162024-02-08T06:41:04.782Z{"msg":"b9c4c1b3-e71d-4d55-8774-66e999c8743a is now active with session: 3182a6a4-cefd-449b-badc-c047fc3002a4","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016692475Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46172024-02-08T06:41:04.782Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016724493Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303,"session_id":"3182a6a4-cefd-449b-badc-c047fc3002a4"}
46182024-02-08T06:41:04.782Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:04.016747701Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303}
46192024-02-08T06:41:04.782ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
46202024-02-08T06:41:04.782ZLoading write count information from file "/tmp/test_repair_verify.data"
46212024-02-08T06:41:04.782ZRun Repair workload
46222024-02-08T06:41:04.782Z01/30 Write block 177 len 40960 data: 5 5 5 5 6 7 5 5 6 6
46232024-02-08T06:41:04.782Z02/30 Write block 534 len 28672 data: 4 3 4 5 4 4 4
46242024-02-08T06:41:04.782Z03/30 Read block 135 len 12288
46252024-02-08T06:41:04.782Z04/30 Read block 522 len 20480
46262024-02-08T06:41:04.782Z05/30 Read block 314 len 8192
46272024-02-08T06:41:04.782Z06/30 Write block 56 len 4096 data: 7
46282024-02-08T06:41:04.782Z07/30 Write block 351 len 40960 data: 7 5 4 5 5 5 6 5 4 4
46292024-02-08T06:41:04.783Z08/30 Write block 375 len 20480 data: 4 5 5 4 4
46302024-02-08T06:41:04.783Z09/30 Write block 158 len 20480 data: 5 4 4 4 6
46312024-02-08T06:41:04.788Z10/30 Read block 206 len 12288
46322024-02-08T06:41:04.788Z11/30 Write block 509 len 12288 data: 5 7 3
46332024-02-08T06:41:04.788Z12/30 Write block 577 len 12288 data: 2 2 3
46342024-02-08T06:41:04.788Z13/30 Read block 197 len 16384
46352024-02-08T06:41:04.788Z14/30 Read block 501 len 4096
46362024-02-08T06:41:04.788Z15/30 Read block 440 len 40960
46372024-02-08T06:41:04.788Z16/30 Write block 334 len 20480 data: 6 5 6 6 6
46382024-02-08T06:41:04.788Z17/30 Read block 577 len 40960
46392024-02-08T06:41:04.788Z18/30 Flush
46402024-02-08T06:41:04.792Z19/30 Write block 401 len 8192 data: 5 5
46412024-02-08T06:41:04.792Z20/30 Read block 312 len 40960
46422024-02-08T06:41:04.792Z21/30 Read block 374 len 40960
46432024-02-08T06:41:04.795Z22/30 Read block 171 len 28672
46442024-02-08T06:41:04.795Z23/30 Write block 195 len 8192 data: 5 5
46452024-02-08T06:41:04.795Z24/30 Read block 471 len 16384
46462024-02-08T06:41:04.795Z25/30 Write block 432 len 24576 data: 3 3 4 4 5 4
46472024-02-08T06:41:04.799Z26/30 Flush
46482024-02-08T06:41:04.799Z27/30 Write block 331 len 36864 data: 3 4 5 7 6 7 7 7 6
46492024-02-08T06:41:04.803Z28/30 Read block 478 len 40960
46502024-02-08T06:41:04.803Z29/30 Write block 436 len 8192 data: 6 5
46512024-02-08T06:41:04.803Z30/30 Read block 416 len 36864
46522024-02-08T06:41:04.803Z----------------------------------------------------------------
46532024-02-08T06:41:04.803Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46542024-02-08T06:41:04.803ZStates: Active Active Active
46552024-02-08T06:41:04.803ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46562024-02-08T06:41:04.803Z 1 Acked 1000 Write 10 Done Done Done false
46572024-02-08T06:41:04.803Z 2 Acked 1001 Write 7 Done Done Done false
46582024-02-08T06:41:04.803Z 3 Acked 1002 Read 3 Done Done Done false
46592024-02-08T06:41:04.803Z 4 Acked 1003 Read 5 Done Done Done false
46602024-02-08T06:41:04.803Z 5 Acked 1004 Read 2 Done Done Done false
46612024-02-08T06:41:04.804Z 6 Acked 1005 Write 1 Done Done Done false
46622024-02-08T06:41:04.804Z 7 Acked 1006 Write 10 Done Done Sent false
46632024-02-08T06:41:04.804Z 8 Acked 1007 Write 5 Done Done Sent false
46642024-02-08T06:41:04.804Z 9 Acked 1008 Write 5 Done Done Sent false
46652024-02-08T06:41:04.804Z 10 Acked 1009 Read 3 Done Done Sent false
46662024-02-08T06:41:04.804Z 11 Acked 1010 Write 3 Done Done Sent false
46672024-02-08T06:41:04.804Z 12 Acked 1011 Write 3 Done Done Sent false
46682024-02-08T06:41:04.804Z 13 Acked 1012 Read 4 Done Done Sent false
46692024-02-08T06:41:04.804Z 14 Acked 1013 Read 1 Done Done Sent false
46702024-02-08T06:41:04.804Z 15 Acked 1014 Read 10 Done Done Sent false
46712024-02-08T06:41:04.804Z 16 Acked 1015 Write 5 Done Done Sent false
46722024-02-08T06:41:04.804Z 17 Acked 1016 Read 10 Done Done Sent false
46732024-02-08T06:41:04.804Z 18 Acked 1017 Flush 0 Done Done Sent false
46742024-02-08T06:41:04.804Z 19 Acked 1018 Write 2 Done Done Sent false
46752024-02-08T06:41:04.804Z 20 Acked 1019 Read 10 Done Done Sent false
46762024-02-08T06:41:04.805Z 21 Acked 1020 Read 10 Done Done Sent false
46772024-02-08T06:41:04.805Z 22 Acked 1021 Read 7 Done Done Sent false
46782024-02-08T06:41:04.805Z 23 Acked 1022 Write 2 Done Done Sent false
46792024-02-08T06:41:04.805Z 24 Acked 1023 Read 4 Done Done Sent false
46802024-02-08T06:41:04.805Z 25 Acked 1024 Write 6 Done Done Sent false
46812024-02-08T06:41:04.805Z 26 Acked 1025 Flush 0 Done Done Sent false
46822024-02-08T06:41:04.805Z 27 Acked 1026 Write 9 Done Done Sent false
46832024-02-08T06:41:04.805Z 28 Acked 1027 Read 10 Done Done Sent false
46842024-02-08T06:41:04.805Z 29 Acked 1028 Write 2 Done Done Sent false
46852024-02-08T06:41:04.805Z 30 Acked 1029 Read 9 Done Sent Sent false
46862024-02-08T06:41:04.805Z STATES DS:0 DS:1 DS:2 TOTAL
46872024-02-08T06:41:04.805Z New 0 0 0 0
46882024-02-08T06:41:04.805Z Sent 0 1 24 25
46892024-02-08T06:41:04.805Z Done 30 29 6 65
46902024-02-08T06:41:04.805Z Skipped 0 0 0 0
46912024-02-08T06:41:04.805Z Error 0 0 0 0
46922024-02-08T06:41:04.806ZLast Flush: 1025 1025 0
46932024-02-08T06:41:04.806ZDownstairs last five completed:
46942024-02-08T06:41:04.806ZUpstairs last five completed: 30 29 28 27 26
46952024-02-08T06:41:04.809Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:04.049099932Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1303}
46962024-02-08T06:41:04.809ZWrote out file "/tmp/test_repair_verify.data"
46972024-02-08T06:41:04.809Z
46982024-02-08T06:41:04.816ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46992024-02-08T06:41:04.826ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47002024-02-08T06:41:04.826Z 0 000-019 40 40 40 62 62 62 F F F
47012024-02-08T06:41:04.826Z 1 020-039 40 40 40 62 62 62 F F F
47022024-02-08T06:41:04.826Z 2 040-059 42 42 27 65 65 45 F F T <---
47032024-02-08T06:41:04.826Z 3 060-079 40 40 40 63 63 63 F F F
47042024-02-08T06:41:04.826Z 4 080-099 40 40 40 60 60 60 F F F
47052024-02-08T06:41:04.826Z 5 100-119 36 36 36 57 57 57 F F F
47062024-02-08T06:41:04.826Z 6 120-139 36 36 36 57 57 57 F F F
47072024-02-08T06:41:04.826Z 7 140-159 42 42 38 65 65 58 F F F <---
47082024-02-08T06:41:04.826Z 8 160-179 42 42 36 65 65 57 F F T <---
47092024-02-08T06:41:04.826Z 9 180-199 42 42 41 66 66 64 F F T <---
47102024-02-08T06:41:04.826Z 10 200-219 41 41 41 64 64 64 F F F
47112024-02-08T06:41:04.830Z 11 220-239 41 41 41 64 64 64 F F F
47122024-02-08T06:41:04.831Z 12 240-259 41 41 41 64 64 64 F F F
47132024-02-08T06:41:04.831Z 13 260-279 41 41 41 64 64 64 F F F
47142024-02-08T06:41:04.831Z 14 280-299 36 36 36 57 57 57 F F F
47152024-02-08T06:41:04.831Z 15 300-319 35 35 35 56 56 56 F F F
47162024-02-08T06:41:04.831Z 16 320-339 42 42 39 65 65 59 T T F <---
47172024-02-08T06:41:04.831Z 17 340-359 42 42 40 65 65 62 F F F <---
47182024-02-08T06:41:04.831Z 18 360-379 42 42 40 65 65 62 F F F <---
47192024-02-08T06:41:04.831Z 19 380-399 39 39 39 59 59 59 F F F
47202024-02-08T06:41:04.831Z 20 400-419 42 42 40 66 66 62 F F F <---
47212024-02-08T06:41:04.831Z 21 420-439 42 42 41 66 66 64 T T F <---
47222024-02-08T06:41:04.831Z 22 440-459 39 39 39 59 59 59 F F F
47232024-02-08T06:41:04.831Z 23 460-479 39 39 39 59 59 59 F F F
47242024-02-08T06:41:04.831Z 24 480-499 41 41 41 64 64 64 F F F
47252024-02-08T06:41:04.831Z 25 500-519 42 42 41 65 65 64 F F F <---
47262024-02-08T06:41:04.832Z 26 520-539 42 42 39 65 65 59 F F T <---
47272024-02-08T06:41:04.832Z 27 540-559 42 42 36 65 65 57 F F T <---
47282024-02-08T06:41:04.832Z 28 560-579 42 42 33 65 65 54 F F F <---
47292024-02-08T06:41:04.832Z 29 580-599 41 41 41 64 64 64 F F F
47302024-02-08T06:41:04.832ZMax gen: 42, Max flush: 66
47312024-02-08T06:41:04.832ZError: Difference in extent metadata found!
47322024-02-08T06:41:04.832ZA Difference in extent metadata is expected here
47332024-02-08T06:41:04.832ZOn loop 21
47342024-02-08T06:41:04.832Z
47352024-02-08T06:41:04.837ZVerifying data now
47362024-02-08T06:41:10.453ZNo change after verify
47372024-02-08T06:41:10.456ZLoop: 21 Downstairs dump after verify (and repair):
47382024-02-08T06:41:10.468ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47392024-02-08T06:41:10.468Z 0 000-019 40 40 40 62 62 62 F F F
47402024-02-08T06:41:10.468Z 1 020-039 40 40 40 62 62 62 F F F
47412024-02-08T06:41:10.468Z 2 040-059 42 42 42 65 65 65 F F F
47422024-02-08T06:41:10.468Z 3 060-079 40 40 40 63 63 63 F F F
47432024-02-08T06:41:10.468Z 4 080-099 40 40 40 60 60 60 F F F
47442024-02-08T06:41:10.469Z 5 100-119 36 36 36 57 57 57 F F F
47452024-02-08T06:41:10.469Z 6 120-139 36 36 36 57 57 57 F F F
47462024-02-08T06:41:10.469Z 7 140-159 42 42 42 65 65 65 F F F
47472024-02-08T06:41:10.475Z 8 160-179 42 42 42 65 65 65 F F F
47482024-02-08T06:41:10.475Z 9 180-199 42 42 42 66 66 66 F F F
47492024-02-08T06:41:10.475Z 10 200-219 41 41 41 64 64 64 F F F
47502024-02-08T06:41:10.475Z 11 220-239 41 41 41 64 64 64 F F F
47512024-02-08T06:41:10.475Z 12 240-259 41 41 41 64 64 64 F F F
47522024-02-08T06:41:10.475Z 13 260-279 41 41 41 64 64 64 F F F
47532024-02-08T06:41:10.475Z 14 280-299 36 36 36 57 57 57 F F F
47542024-02-08T06:41:10.475Z 15 300-319 35 35 35 56 56 56 F F F
47552024-02-08T06:41:10.475Z 16 320-339 43 43 43 67 67 67 F F F
47562024-02-08T06:41:10.475Z 17 340-359 42 42 42 65 65 65 F F F
47572024-02-08T06:41:10.475Z 18 360-379 42 42 42 65 65 65 F F F
47582024-02-08T06:41:10.475Z 19 380-399 39 39 39 59 59 59 F F F
47592024-02-08T06:41:10.476Z 20 400-419 42 42 42 66 66 66 F F F
47602024-02-08T06:41:10.476Z 21 420-439 43 43 43 67 67 67 F F F
47612024-02-08T06:41:10.476Z 22 440-459 39 39 39 59 59 59 F F F
47622024-02-08T06:41:10.476Z 23 460-479 39 39 39 59 59 59 F F F
47632024-02-08T06:41:10.476Z 24 480-499 41 41 41 64 64 64 F F F
47642024-02-08T06:41:10.476Z 25 500-519 42 42 42 65 65 65 F F F
47652024-02-08T06:41:10.476Z 26 520-539 42 42 42 65 65 65 F F F
47662024-02-08T06:41:10.476Z 27 540-559 42 42 42 65 65 65 F F F
47672024-02-08T06:41:10.476Z 28 560-579 42 42 42 65 65 65 F F F
47682024-02-08T06:41:10.476Z 29 580-599 41 41 41 64 64 64 F F F
47692024-02-08T06:41:10.476ZMax gen: 43, Max flush: 67
47702024-02-08T06:41:10.476Z
47712024-02-08T06:41:10.476ZBegin loop 22
47722024-02-08T06:41:10.476ZDownstairs to restart: 0
47732024-02-08T06:41:10.489Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.73522177Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311}
47742024-02-08T06:41:10.489Z{Crucible runtime is spawned
47752024-02-08T06:41:10.493Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735501989Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311}
47762024-02-08T06:41:10.493Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735545356Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311}
47772024-02-08T06:41:10.493Z{"msg":"Crucible 550c56c1-e7f4-4fd7-8a80-b46ba9b18830 has session id: 7312f1bf-8385-4080-baeb-f23c0bac238d","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735570054Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47782024-02-08T06:41:10.493Z{"msg":"Upstairs opts: Upstairs UUID: 550c56c1-e7f4-4fd7-8a80-b46ba9b18830, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735612091Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47792024-02-08T06:41:10.494Z{"msg":"Crucible stats registered with UUID: 550c56c1-e7f4-4fd7-8a80-b46ba9b18830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735637989Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47802024-02-08T06:41:10.494Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735671847Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311}
47812024-02-08T06:41:10.494Z{"msg":"550c56c1-e7f4-4fd7-8a80-b46ba9b18830 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735715963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47822024-02-08T06:41:10.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735741722Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47832024-02-08T06:41:10.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735784188Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47842024-02-08T06:41:10.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735823075Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47852024-02-08T06:41:10.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735842014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47862024-02-08T06:41:10.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735867412Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47872024-02-08T06:41:10.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.73589217Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47882024-02-08T06:41:10.494Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735937897Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47892024-02-08T06:41:10.494Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735974604Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47902024-02-08T06:41:10.494Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.735996532Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47912024-02-08T06:41:10.494Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.736013711Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47922024-02-08T06:41:10.494Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:09.736121313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47932024-02-08T06:41:10.495Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:09.736174559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47942024-02-08T06:41:10.495Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:09.736207236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47952024-02-08T06:41:10.495Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.736242714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47962024-02-08T06:41:10.495Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.736268812Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47972024-02-08T06:41:10.495Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.736307509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47982024-02-08T06:41:10.495Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.736341996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
47992024-02-08T06:41:10.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.736785763Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48002024-02-08T06:41:10.495Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.73682388Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48012024-02-08T06:41:10.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.736928672Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48022024-02-08T06:41:10.495Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.73695653Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48032024-02-08T06:41:10.495Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.737310624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48042024-02-08T06:41:10.495Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.737446303Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48052024-02-08T06:41:10.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.737809646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48062024-02-08T06:41:10.495Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.737875521Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48072024-02-08T06:41:10.495Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.737911779Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48082024-02-08T06:41:10.496Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.737927487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48092024-02-08T06:41:10.496Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.737948896Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48102024-02-08T06:41:10.496Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:09.737963185Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48112024-02-08T06:41:11.492Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.737776811Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48122024-02-08T06:41:11.500Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.738009963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48132024-02-08T06:41:11.500Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.738835161Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48142024-02-08T06:41:11.500Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.738873958Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48152024-02-08T06:41:11.500Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.739369771Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48162024-02-08T06:41:11.500Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.739922439Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48172024-02-08T06:41:11.500Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.739959627Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48182024-02-08T06:41:11.500Z{"msg":"[0]R flush_numbers[0..12]: [62, 62, 65, 63, 60, 57, 57, 65, 65, 66, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.739976985Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48192024-02-08T06:41:11.500Z{"msg":"[0]R generation[0..12]: [40, 40, 42, 40, 40, 36, 36, 42, 42, 42, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740003173Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48202024-02-08T06:41:11.500Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740020332Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48212024-02-08T06:41:11.500Z{"msg":"[1]R flush_numbers[0..12]: [62, 62, 65, 63, 60, 57, 57, 65, 65, 66, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740035041Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48222024-02-08T06:41:11.500Z{"msg":"[1]R generation[0..12]: [40, 40, 42, 40, 40, 36, 36, 42, 42, 42, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740056029Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48232024-02-08T06:41:11.500Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740070318Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48242024-02-08T06:41:11.500Z{"msg":"[2]R flush_numbers[0..12]: [62, 62, 65, 63, 60, 57, 57, 65, 65, 66, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740090677Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48252024-02-08T06:41:11.500Z{"msg":"[2]R generation[0..12]: [40, 40, 42, 40, 40, 36, 36, 42, 42, 42, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740104646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48262024-02-08T06:41:11.501Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740119495Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48272024-02-08T06:41:11.501Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740133043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48282024-02-08T06:41:11.501ZWait for a query_work_queue command to finish before sending IO
48292024-02-08T06:41:11.501Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740162761Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48302024-02-08T06:41:11.501Z{"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.74017598Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48312024-02-08T06:41:11.501Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740190769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48322024-02-08T06:41:11.501Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740204488Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48332024-02-08T06:41:11.501Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740216717Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48342024-02-08T06:41:11.501Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740236696Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48352024-02-08T06:41:11.501Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740250385Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48362024-02-08T06:41:11.501Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740263854Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48372024-02-08T06:41:11.501Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740283652Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48382024-02-08T06:41:11.501Z{"msg":"550c56c1-e7f4-4fd7-8a80-b46ba9b18830 is now active with session: 7312f1bf-8385-4080-baeb-f23c0bac238d","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.740296771Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48392024-02-08T06:41:11.501Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.74030885Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311,"session_id":"7312f1bf-8385-4080-baeb-f23c0bac238d"}
48402024-02-08T06:41:11.501Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:10.74032134Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311}
48412024-02-08T06:41:11.501ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48422024-02-08T06:41:11.502ZLoading write count information from file "/tmp/test_repair_verify.data"
48432024-02-08T06:41:11.502ZRun Repair workload
48442024-02-08T06:41:11.502Z01/30 Write block 114 len 8192 data: 4 4
48452024-02-08T06:41:11.505Z02/30 Write block 525 len 24576 data: 4 4 3 4 4 5
48462024-02-08T06:41:11.505Z03/30 Read block 424 len 40960
48472024-02-08T06:41:11.505Z04/30 Write block 356 len 40960 data: 6 7 6 5 5 3 3 2 3 4
48482024-02-08T06:41:11.505Z05/30 Read block 86 len 12288
48492024-02-08T06:41:11.505Z06/30 Read block 156 len 12288
48502024-02-08T06:41:11.505Z07/30 Read block 474 len 28672
48512024-02-08T06:41:11.505Z08/30 Write block 527 len 36864 data: 4 5 5 6 4 4 5 5 4
48522024-02-08T06:41:11.506Z09/30 Read block 254 len 16384
48532024-02-08T06:41:11.506Z10/30 Write block 589 len 32768 data: 5 4 4 4 4 4 2 2
48542024-02-08T06:41:11.509Z11/30 Write block 21 len 40960 data: 4 5 4 5 6 5 5 5 4 5
48552024-02-08T06:41:11.509Z12/30 Read block 219 len 20480
48562024-02-08T06:41:11.509Z13/30 Write block 248 len 40960 data: 7 10 9 9 6 5 5 5 5 4
48572024-02-08T06:41:11.509Z14/30 Read block 539 len 24576
48582024-02-08T06:41:11.512Z15/30 Read block 331 len 32768
48592024-02-08T06:41:11.512Z16/30 Write block 20 len 32768 data: 5 5 6 5 6 7 6 6
48602024-02-08T06:41:11.512Z17/30 Write block 219 len 12288 data: 6 7 8
48612024-02-08T06:41:11.515Z18/30 Write block 583 len 32768 data: 4 4 2 2 3 4 6 5
48622024-02-08T06:41:11.515Z19/30 Read block 476 len 40960
48632024-02-08T06:41:11.515Z20/30 Write block 434 len 28672 data: 5 5 7 6 4 3 5
48642024-02-08T06:41:11.518Z21/30 Write block 78 len 12288 data: 2 2 3
48652024-02-08T06:41:11.518Z22/30 Read block 534 len 28672
48662024-02-08T06:41:11.518Z23/30 Read block 149 len 40960
48672024-02-08T06:41:11.518Z24/30 Read block 323 len 16384
48682024-02-08T06:41:11.522Z25/30 Read block 311 len 32768
48692024-02-08T06:41:11.522Z26/30 Read block 325 len 12288
48702024-02-08T06:41:11.522Z27/30 Write block 370 len 12288 data: 5 5 6
48712024-02-08T06:41:11.522Z28/30 Read block 319 len 16384
48722024-02-08T06:41:11.525Z29/30 Write block 310 len 20480 data: 3 3 3 4 4
48732024-02-08T06:41:11.525Z30/30 Read block 535 len 36864
48742024-02-08T06:41:11.526Z----------------------------------------------------------------
48752024-02-08T06:41:11.526Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48762024-02-08T06:41:11.526ZStates: Active Active Active
48772024-02-08T06:41:11.526ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48782024-02-08T06:41:11.526Z 1 Acked 1000 Write 2 Done Done Done false
48792024-02-08T06:41:11.526Z 2 Acked 1001 Write 6 Done Done Done false
48802024-02-08T06:41:11.526Z 3 Acked 1002 Read 10 Sent Done Done false
48812024-02-08T06:41:11.526Z 4 Acked 1003 Write 10 Sent Done Done false
48822024-02-08T06:41:11.526Z 5 Acked 1004 Read 3 Sent Done Done false
48832024-02-08T06:41:11.526Z 6 Acked 1005 Read 3 Sent Done Done false
48842024-02-08T06:41:11.526Z 7 Acked 1006 Read 7 Sent Done Done false
48852024-02-08T06:41:11.526Z 8 Acked 1007 Write 9 Sent Done Done false
48862024-02-08T06:41:11.526Z 9 Acked 1008 Read 4 Sent Done Done false
48872024-02-08T06:41:11.526Z 10 Acked 1009 Write 8 Sent Done Done false
48882024-02-08T06:41:11.527Z 11 Acked 1010 Write 10 Sent Done Done false
48892024-02-08T06:41:11.527Z 12 Acked 1011 Read 5 Sent Done Done false
48902024-02-08T06:41:11.527Z 13 Acked 1012 Write 10 Sent Done Done false
48912024-02-08T06:41:11.527Z 14 Acked 1013 Read 6 Sent Done Done false
48922024-02-08T06:41:11.527Z 15 Acked 1014 Read 8 Sent Done Done false
48932024-02-08T06:41:11.527Z 16 Acked 1015 Write 8 Sent Done Done false
48942024-02-08T06:41:11.527Z 17 Acked 1016 Write 3 Sent Done Done false
48952024-02-08T06:41:11.527Z 18 Acked 1017 Write 8 Sent Done Done false
48962024-02-08T06:41:11.527Z 19 Acked 1018 Read 10 Sent Done Done false
48972024-02-08T06:41:11.527Z 20 Acked 1019 Write 7 Sent Done Done false
48982024-02-08T06:41:11.527Z 21 Acked 1020 Write 3 Sent Done Done false
48992024-02-08T06:41:11.527Z 22 Acked 1021 Read 7 Sent Done Done false
49002024-02-08T06:41:11.527Z 23 Acked 1022 Read 10 Sent Done Done false
49012024-02-08T06:41:11.527Z 24 Acked 1023 Read 4 Sent Done Done false
49022024-02-08T06:41:11.527Z 25 Acked 1024 Read 8 Sent Done Done false
49032024-02-08T06:41:11.527Z 26 Acked 1025 Read 3 Sent Done Done false
49042024-02-08T06:41:11.528Z 27 Acked 1026 Write 3 Sent Done Done false
49052024-02-08T06:41:11.528Z 28 Acked 1027 Read 4 Sent Done Done false
49062024-02-08T06:41:11.528Z 29 Acked 1028 Write 5 Sent Done Done false
49072024-02-08T06:41:11.528Z 30 Acked 1029 Read 9 Sent Sent Done false
49082024-02-08T06:41:11.528Z STATES DS:0 DS:1 DS:2 TOTAL
49092024-02-08T06:41:11.528Z New 0 0 0 0
49102024-02-08T06:41:11.528Z Sent 28 1 0 29
49112024-02-08T06:41:11.528Z Done 2 29 30 61
49122024-02-08T06:41:11.528Z Skipped 0 0 0 0
49132024-02-08T06:41:11.528Z Error 0 0 0 0
49142024-02-08T06:41:11.528ZLast Flush: 0 0 0
49152024-02-08T06:41:11.528ZDownstairs last five completed:
49162024-02-08T06:41:11.528ZUpstairs last five completed: 30 29 28 27 26
49172024-02-08T06:41:11.528Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:10.770729977Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1311}
49182024-02-08T06:41:11.528ZWrote out file "/tmp/test_repair_verify.data"
49192024-02-08T06:41:11.532Z
49202024-02-08T06:41:11.534ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
49212024-02-08T06:41:11.554ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49222024-02-08T06:41:11.554Z 0 000-019 40 40 40 62 62 62 F F F
49232024-02-08T06:41:11.554Z 1 020-039 40 40 40 62 62 62 F T T <---
49242024-02-08T06:41:11.554Z 2 040-059 42 42 42 65 65 65 F F F
49252024-02-08T06:41:11.554Z 3 060-079 40 40 40 63 63 63 F T T <---
49262024-02-08T06:41:11.554Z 4 080-099 40 40 40 60 60 60 F T T <---
49272024-02-08T06:41:11.557Z 5 100-119 36 36 36 57 57 57 T T T
49282024-02-08T06:41:11.557Z 6 120-139 36 36 36 57 57 57 F F F
49292024-02-08T06:41:11.557Z 7 140-159 42 42 42 65 65 65 F F F
49302024-02-08T06:41:11.557Z 8 160-179 42 42 42 65 65 65 F F F
49312024-02-08T06:41:11.557Z 9 180-199 42 42 42 66 66 66 F F F
49322024-02-08T06:41:11.558Z 10 200-219 41 41 41 64 64 64 F T T <---
49332024-02-08T06:41:11.558Z 11 220-239 41 41 41 64 64 64 F T T <---
49342024-02-08T06:41:11.558Z 12 240-259 41 41 41 64 64 64 F T T <---
49352024-02-08T06:41:11.558Z 13 260-279 41 41 41 64 64 64 F F F
49362024-02-08T06:41:11.558Z 14 280-299 36 36 36 57 57 57 F F F
49372024-02-08T06:41:11.558Z 15 300-319 35 35 35 56 56 56 F T T <---
49382024-02-08T06:41:11.558Z 16 320-339 43 43 43 67 67 67 F F F
49392024-02-08T06:41:11.558Z 17 340-359 42 42 42 65 65 65 F T T <---
49402024-02-08T06:41:11.558Z 18 360-379 42 42 42 65 65 65 F T T <---
49412024-02-08T06:41:11.558Z 19 380-399 39 39 39 59 59 59 F F F
49422024-02-08T06:41:11.558Z 20 400-419 42 42 42 66 66 66 F F F
49432024-02-08T06:41:11.558Z 21 420-439 43 43 43 67 67 67 F T T <---
49442024-02-08T06:41:11.558Z 22 440-459 39 39 39 59 59 59 F T T <---
49452024-02-08T06:41:11.558Z 23 460-479 39 39 39 59 59 59 F F F
49462024-02-08T06:41:11.558Z 24 480-499 41 41 41 64 64 64 F F F
49472024-02-08T06:41:11.559Z 25 500-519 42 42 42 65 65 65 F F F
49482024-02-08T06:41:11.559Z 26 520-539 42 42 42 65 65 65 T T T
49492024-02-08T06:41:11.559Z 27 540-559 42 42 42 65 65 65 F F F
49502024-02-08T06:41:11.559Z 28 560-579 42 42 42 65 65 65 F F F
49512024-02-08T06:41:11.559Z 29 580-599 41 41 41 64 64 64 F T T <---
49522024-02-08T06:41:11.559ZMax gen: 43, Max flush: 67
49532024-02-08T06:41:11.559ZError: Difference in extent metadata found!
49542024-02-08T06:41:11.559ZA Difference in extent metadata is expected here
49552024-02-08T06:41:11.559ZOn loop 22
49562024-02-08T06:41:11.559Z
49572024-02-08T06:41:11.562ZVerifying data now
49582024-02-08T06:41:17.310ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
49592024-02-08T06:41:17.314Z530d529
49602024-02-08T06:41:17.314Z< 3,
49612024-02-08T06:41:17.314Z532d530
49622024-02-08T06:41:17.314Z< 4,
49632024-02-08T06:41:17.314Z534,535c532,533
49642024-02-08T06:41:17.314Z< 3,
49652024-02-08T06:41:17.314Z< 3,
49662024-02-08T06:41:17.314Z---
49672024-02-08T06:41:17.314Z> 5,
49682024-02-08T06:41:17.314Z> 6,
49692024-02-08T06:41:17.315Z538c536,537
49702024-02-08T06:41:17.315Z< 3,
49712024-02-08T06:41:17.315Z---
49722024-02-08T06:41:17.315Z> 5,
49732024-02-08T06:41:17.315Z> 5,
49742024-02-08T06:41:17.315Z539a539
49752024-02-08T06:41:17.315Z> 4,
49762024-02-08T06:41:17.315Zdiff found after verify
49772024-02-08T06:41:17.319ZLoop: 22 Downstairs dump after verify (and repair):
49782024-02-08T06:41:17.331ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49792024-02-08T06:41:17.332Z 0 000-019 40 40 40 62 62 62 F F F
49802024-02-08T06:41:17.332Z 1 020-039 44 44 44 68 68 68 F F F
49812024-02-08T06:41:17.332Z 2 040-059 42 42 42 65 65 65 F F F
49822024-02-08T06:41:17.332Z 3 060-079 44 44 44 68 68 68 F F F
49832024-02-08T06:41:17.332Z 4 080-099 44 44 44 68 68 68 F F F
49842024-02-08T06:41:17.332Z 5 100-119 44 44 44 68 68 68 F F F
49852024-02-08T06:41:17.332Z 6 120-139 36 36 36 57 57 57 F F F
49862024-02-08T06:41:17.332Z 7 140-159 42 42 42 65 65 65 F F F
49872024-02-08T06:41:17.332Z 8 160-179 42 42 42 65 65 65 F F F
49882024-02-08T06:41:17.332Z 9 180-199 42 42 42 66 66 66 F F F
49892024-02-08T06:41:17.336Z 10 200-219 44 44 44 68 68 68 F F F
49902024-02-08T06:41:17.336Z 11 220-239 44 44 44 68 68 68 F F F
49912024-02-08T06:41:17.336Z 12 240-259 44 44 44 68 68 68 F F F
49922024-02-08T06:41:17.336Z 13 260-279 41 41 41 64 64 64 F F F
49932024-02-08T06:41:17.336Z 14 280-299 36 36 36 57 57 57 F F F
49942024-02-08T06:41:17.336Z 15 300-319 44 44 44 68 68 68 F F F
49952024-02-08T06:41:17.337Z 16 320-339 43 43 43 67 67 67 F F F
49962024-02-08T06:41:17.337Z 17 340-359 44 44 44 68 68 68 F F F
49972024-02-08T06:41:17.337Z 18 360-379 44 44 44 68 68 68 F F F
49982024-02-08T06:41:17.337Z 19 380-399 39 39 39 59 59 59 F F F
49992024-02-08T06:41:17.337Z 20 400-419 42 42 42 66 66 66 F F F
50002024-02-08T06:41:17.337Z 21 420-439 44 44 44 68 68 68 F F F
50012024-02-08T06:41:17.337Z 22 440-459 44 44 44 68 68 68 F F F
50022024-02-08T06:41:17.337Z 23 460-479 39 39 39 59 59 59 F F F
50032024-02-08T06:41:17.337Z 24 480-499 41 41 41 64 64 64 F F F
50042024-02-08T06:41:17.338Z 25 500-519 42 42 42 65 65 65 F F F
50052024-02-08T06:41:17.338Z 26 520-539 44 44 44 68 68 68 F F F
50062024-02-08T06:41:17.338Z 27 540-559 42 42 42 65 65 65 F F F
50072024-02-08T06:41:17.338Z 28 560-579 42 42 42 65 65 65 F F F
50082024-02-08T06:41:17.338Z 29 580-599 44 44 44 68 68 68 F F F
50092024-02-08T06:41:17.338ZMax gen: 44, Max flush: 68
50102024-02-08T06:41:17.338Z
50112024-02-08T06:41:17.338ZBegin loop 23
50122024-02-08T06:41:17.339ZDownstairs to restart: 2
50132024-02-08T06:41:17.351Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.596761241Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322}
50142024-02-08T06:41:17.352Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597001643Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322}
50152024-02-08T06:41:17.357Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597052229Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322}
50162024-02-08T06:41:17.357Z{Crucible runtime is spawned
50172024-02-08T06:41:17.357Z"msg":"Crucible 64d28875-dbf7-4838-88e5-9e359df25661 has session id: f13d6fba-1e65-461e-a18c-d0d0b7509770","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597095486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50182024-02-08T06:41:17.357Z{"msg":"Upstairs opts: Upstairs UUID: 64d28875-dbf7-4838-88e5-9e359df25661, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597128734Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50192024-02-08T06:41:17.357Z{"msg":"Crucible stats registered with UUID: 64d28875-dbf7-4838-88e5-9e359df25661","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597166261Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50202024-02-08T06:41:17.357Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597201638Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322}
50212024-02-08T06:41:17.357Z{"msg":"64d28875-dbf7-4838-88e5-9e359df25661 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.59731216Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50222024-02-08T06:41:17.357Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597354537Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50232024-02-08T06:41:17.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597387184Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50242024-02-08T06:41:17.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597428421Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50252024-02-08T06:41:17.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.59744894Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50262024-02-08T06:41:17.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597470908Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50272024-02-08T06:41:17.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597485037Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50282024-02-08T06:41:17.358Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597518254Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"io task","client":"2","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50292024-02-08T06:41:17.358Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597540963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"io task","client":"0","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50302024-02-08T06:41:17.358Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.59757546Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"io task","client":"1","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50312024-02-08T06:41:17.358Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597601958Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"io task","client":"2","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50322024-02-08T06:41:17.358Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597787804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"io task","client":"1","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50332024-02-08T06:41:17.358Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.597825751Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"io task","client":"0","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50342024-02-08T06:41:17.358Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.598390349Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50352024-02-08T06:41:17.358Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.598438285Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50362024-02-08T06:41:17.359Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.598498031Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50372024-02-08T06:41:17.359Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.598527139Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50382024-02-08T06:41:17.359Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.599022522Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50392024-02-08T06:41:17.359Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.599064818Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50402024-02-08T06:41:17.359Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.599483877Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50412024-02-08T06:41:17.359Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.599553731Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50422024-02-08T06:41:17.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.599683422Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50432024-02-08T06:41:17.359Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.599729588Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50442024-02-08T06:41:17.359Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.599802453Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50452024-02-08T06:41:17.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.599856479Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50462024-02-08T06:41:17.359Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.599909875Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50472024-02-08T06:41:17.359Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.599983679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50482024-02-08T06:41:17.359Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.60010527Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50492024-02-08T06:41:17.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600590504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50502024-02-08T06:41:17.360Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600630101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50512024-02-08T06:41:17.360Z{"msg":"[0]R flush_numbers[0..12]: [62, 68, 65, 68, 68, 68, 57, 65, 65, 66, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600647369Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50522024-02-08T06:41:17.360Z{"msg":"[0]R generation[0..12]: [40, 44, 42, 44, 44, 44, 36, 42, 42, 42, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600669668Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50532024-02-08T06:41:17.360Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600684717Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50542024-02-08T06:41:17.360Z{"msg":"[1]R flush_numbers[0..12]: [62, 68, 65, 68, 68, 68, 57, 65, 65, 66, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600706735Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50552024-02-08T06:41:17.360Z{"msg":"[1]R generation[0..12]: [40, 44, 42, 44, 44, 44, 36, 42, 42, 42, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600721624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50562024-02-08T06:41:17.360Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600743382Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50572024-02-08T06:41:17.360Z{"msg":"[2]R flush_numbers[0..12]: [62, 68, 65, 68, 68, 68, 57, 65, 65, 66, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600758151Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50582024-02-08T06:41:17.360Z{"msg":"[2]R generation[0..12]: [40, 44, 42, 44, 44, 44, 36, 42, 42, 42, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.60077227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50592024-02-08T06:41:17.360ZWait for a query_work_queue command to finish before sending IO
50602024-02-08T06:41:17.364Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600803568Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50612024-02-08T06:41:17.365Z{"msg":"Max found gen is 45","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600819247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50622024-02-08T06:41:17.365Z{"msg":"Generation requested: 46 >= found:45","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600833106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50632024-02-08T06:41:17.365Z{"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600854194Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50642024-02-08T06:41:17.365Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600868933Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50652024-02-08T06:41:17.365Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600890641Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50662024-02-08T06:41:17.365Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.60090331Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50672024-02-08T06:41:17.365Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600923829Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50682024-02-08T06:41:17.365Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600937988Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50692024-02-08T06:41:17.365Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600951587Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50702024-02-08T06:41:17.365Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600964206Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50712024-02-08T06:41:17.365Z{"msg":"64d28875-dbf7-4838-88e5-9e359df25661 is now active with session: f13d6fba-1e65-461e-a18c-d0d0b7509770","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600977695Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50722024-02-08T06:41:17.365Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.600989694Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322,"session_id":"f13d6fba-1e65-461e-a18c-d0d0b7509770"}
50732024-02-08T06:41:17.365Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:16.601003053Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322}
50742024-02-08T06:41:17.366ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50752024-02-08T06:41:17.366ZLoading write count information from file "/tmp/test_repair_verify.data"
50762024-02-08T06:41:17.366ZRun Repair workload
50772024-02-08T06:41:17.366Z01/30 Flush
50782024-02-08T06:41:17.366Z02/30 Write block 115 len 8192 data: 5 3
50792024-02-08T06:41:17.366Z03/30 Read block 431 len 16384
50802024-02-08T06:41:17.366Z04/30 Write block 151 len 4096 data: 5
50812024-02-08T06:41:17.366Z05/30 Write block 527 len 20480 data: 4 5 5 6 4
50822024-02-08T06:41:17.366Z06/30 Write block 259 len 12288 data: 4 5 6
50832024-02-08T06:41:17.366Z07/30 Read block 314 len 8192
50842024-02-08T06:41:17.366Z08/30 Read block 16 len 12288
50852024-02-08T06:41:17.366Z09/30 Write block 418 len 36864 data: 6 5 6 5 6 6 6 5 4
50862024-02-08T06:41:17.366Z10/30 Read block 275 len 16384
50872024-02-08T06:41:17.370Z11/30 Read block 243 len 36864
50882024-02-08T06:41:17.370Z12/30 Read block 52 len 28672
50892024-02-08T06:41:17.370Z13/30 Flush
50902024-02-08T06:41:17.371Z14/30 Write block 224 len 28672 data: 9 8 8 7 9 9 7
50912024-02-08T06:41:17.371Z15/30 Write block 113 len 20480 data: 4 5 6 4 3
50922024-02-08T06:41:17.371Z16/30 Read block 228 len 32768
50932024-02-08T06:41:17.374Z17/30 Read block 287 len 24576
50942024-02-08T06:41:17.374Z18/30 Write block 73 len 36864 data: 6 6 4 4 2 3 3 4 3
50952024-02-08T06:41:17.374Z19/30 Read block 499 len 8192
50962024-02-08T06:41:17.374Z20/30 Read block 198 len 24576
50972024-02-08T06:41:17.377Z21/30 Read block 152 len 20480
50982024-02-08T06:41:17.377Z22/30 Read block 505 len 16384
50992024-02-08T06:41:17.377Z23/30 Write block 238 len 32768 data: 4 5 4 5 5 7 6 6
51002024-02-08T06:41:17.377Z24/30 Flush
51012024-02-08T06:41:17.381Z25/30 Write block 370 len 12288 data: 6 6 7
51022024-02-08T06:41:17.381Z26/30 Write block 166 len 16384 data: 5 5 4 4
51032024-02-08T06:41:17.384Z27/30 Write block 553 len 20480 data: 5 5 4 7 5
51042024-02-08T06:41:17.384Z28/30 Write block 540 len 20480 data: 5 4 3 3 3
51052024-02-08T06:41:17.385Z29/30 Read block 563 len 36864
51062024-02-08T06:41:17.385Z30/30 Read block 508 len 4096
51072024-02-08T06:41:17.385Z----------------------------------------------------------------
51082024-02-08T06:41:17.385Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:17
51092024-02-08T06:41:17.385ZStates: Active Active Active
51102024-02-08T06:41:17.385ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
51112024-02-08T06:41:17.385Z 14 Acked 1013 Write 7 Done Done Sent false
51122024-02-08T06:41:17.385Z 15 Acked 1014 Write 5 Done Done Sent false
51132024-02-08T06:41:17.385Z 16 Acked 1015 Read 8 Done Done Sent false
51142024-02-08T06:41:17.385Z 17 Acked 1016 Read 6 Done Done Sent false
51152024-02-08T06:41:17.385Z 18 Acked 1017 Write 9 Done Done Sent false
51162024-02-08T06:41:17.386Z 19 Acked 1018 Read 2 Done Done Sent false
51172024-02-08T06:41:17.386Z 20 Acked 1019 Read 6 Done Done Sent false
51182024-02-08T06:41:17.386Z 21 Acked 1020 Read 5 Done Done Sent false
51192024-02-08T06:41:17.386Z 22 Acked 1021 Read 4 Done Done Sent false
51202024-02-08T06:41:17.386Z 23 Acked 1022 Write 8 Done Done Sent false
51212024-02-08T06:41:17.386Z 24 Acked 1023 Flush 0 Done Done Sent false
51222024-02-08T06:41:17.386Z 25 Acked 1024 Write 3 Done Done Sent false
51232024-02-08T06:41:17.386Z 26 Acked 1025 Write 4 Done Done Sent false
51242024-02-08T06:41:17.386Z 27 Acked 1026 Write 5 Done Done Sent false
51252024-02-08T06:41:17.386Z 28 Acked 1027 Write 5 Done Done Sent false
51262024-02-08T06:41:17.386Z 29 Acked 1028 Read 9 Done Done Sent false
51272024-02-08T06:41:17.386Z 30 Acked 1029 Read 1 Done Done Sent false
51282024-02-08T06:41:17.386Z STATES DS:0 DS:1 DS:2 TOTAL
51292024-02-08T06:41:17.386Z New 0 0 0 0
51302024-02-08T06:41:17.387Z Sent 0 0 17 17
51312024-02-08T06:41:17.387Z Done 17 17 0 34
51322024-02-08T06:41:17.387Z Skipped 0 0 0 0
51332024-02-08T06:41:17.387Z Error 0 0 0 0
51342024-02-08T06:41:17.387ZLast Flush: 1023 1023 1012
51352024-02-08T06:41:17.387ZDownstairs last five completed: 1012 1011 1010 1009 1008
51362024-02-08T06:41:17.387ZUpstairs last five completed: 30 29 28 27 26
51372024-02-08T06:41:17.387Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:16.629468816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1322}
51382024-02-08T06:41:17.391ZWrote out file "/tmp/test_repair_verify.data"
51392024-02-08T06:41:17.391Z
51402024-02-08T06:41:17.394ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51412024-02-08T06:41:17.406ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51422024-02-08T06:41:17.406Z 0 000-019 40 40 40 62 62 62 F F F
51432024-02-08T06:41:17.406Z 1 020-039 44 44 44 68 68 68 F F F
51442024-02-08T06:41:17.406Z 2 040-059 42 42 42 65 65 65 F F F
51452024-02-08T06:41:17.406Z 3 060-079 46 46 44 71 71 68 F F F <---
51462024-02-08T06:41:17.410Z 4 080-099 46 46 44 71 71 68 F F F <---
51472024-02-08T06:41:17.410Z 5 100-119 46 46 46 71 71 70 F F F <---
51482024-02-08T06:41:17.410Z 6 120-139 36 36 36 57 57 57 F F F
51492024-02-08T06:41:17.410Z 7 140-159 46 46 46 70 70 70 F F F
51502024-02-08T06:41:17.410Z 8 160-179 42 42 42 65 65 65 T T F <---
51512024-02-08T06:41:17.410Z 9 180-199 42 42 42 66 66 66 F F F
51522024-02-08T06:41:17.410Z 10 200-219 44 44 44 68 68 68 F F F
51532024-02-08T06:41:17.410Z 11 220-239 46 46 44 71 71 68 F F F <---
51542024-02-08T06:41:17.410Z 12 240-259 46 46 46 71 71 70 F F F <---
51552024-02-08T06:41:17.410Z 13 260-279 46 46 46 70 70 70 F F F
51562024-02-08T06:41:17.411Z 14 280-299 36 36 36 57 57 57 F F F
51572024-02-08T06:41:17.411Z 15 300-319 44 44 44 68 68 68 F F F
51582024-02-08T06:41:17.411Z 16 320-339 43 43 43 67 67 67 F F F
51592024-02-08T06:41:17.411Z 17 340-359 44 44 44 68 68 68 F F F
51602024-02-08T06:41:17.411Z 18 360-379 44 44 44 68 68 68 T T F <---
51612024-02-08T06:41:17.411Z 19 380-399 39 39 39 59 59 59 F F F
51622024-02-08T06:41:17.411Z 20 400-419 46 46 46 70 70 70 F F F
51632024-02-08T06:41:17.411Z 21 420-439 46 46 46 70 70 70 F F F
51642024-02-08T06:41:17.411Z 22 440-459 44 44 44 68 68 68 F F F
51652024-02-08T06:41:17.411Z 23 460-479 39 39 39 59 59 59 F F F
51662024-02-08T06:41:17.411Z 24 480-499 41 41 41 64 64 64 F F F
51672024-02-08T06:41:17.411Z 25 500-519 42 42 42 65 65 65 F F F
51682024-02-08T06:41:17.411Z 26 520-539 46 46 46 70 70 70 F F F
51692024-02-08T06:41:17.411Z 27 540-559 42 42 42 65 65 65 T T F <---
51702024-02-08T06:41:17.411Z 28 560-579 42 42 42 65 65 65 F F F
51712024-02-08T06:41:17.412Z 29 580-599 44 44 44 68 68 68 F F F
51722024-02-08T06:41:17.412ZMax gen: 46, Max flush: 71
51732024-02-08T06:41:17.412ZError: Difference in extent metadata found!
51742024-02-08T06:41:17.412ZA Difference in extent metadata is expected here
51752024-02-08T06:41:17.412ZOn loop 23
51762024-02-08T06:41:17.412Z
51772024-02-08T06:41:17.415ZVerifying data now
51782024-02-08T06:41:21.839ZNo change after verify
51792024-02-08T06:41:21.839ZLoop: 23 Downstairs dump after verify (and repair):
51802024-02-08T06:41:21.852ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51812024-02-08T06:41:21.852Z 0 000-019 40 40 40 62 62 62 F F F
51822024-02-08T06:41:21.852Z 1 020-039 44 44 44 68 68 68 F F F
51832024-02-08T06:41:21.852Z 2 040-059 42 42 42 65 65 65 F F F
51842024-02-08T06:41:21.852Z 3 060-079 46 46 46 71 71 71 F F F
51852024-02-08T06:41:21.852Z 4 080-099 46 46 46 71 71 71 F F F
51862024-02-08T06:41:21.852Z 5 100-119 46 46 46 71 71 71 F F F
51872024-02-08T06:41:21.852Z 6 120-139 36 36 36 57 57 57 F F F
51882024-02-08T06:41:21.856Z 7 140-159 46 46 46 70 70 70 F F F
51892024-02-08T06:41:21.856Z 8 160-179 47 47 47 72 72 72 F F F
51902024-02-08T06:41:21.856Z 9 180-199 42 42 42 66 66 66 F F F
51912024-02-08T06:41:21.856Z 10 200-219 44 44 44 68 68 68 F F F
51922024-02-08T06:41:21.856Z 11 220-239 46 46 46 71 71 71 F F F
51932024-02-08T06:41:21.856Z 12 240-259 46 46 46 71 71 71 F F F
51942024-02-08T06:41:21.856Z 13 260-279 46 46 46 70 70 70 F F F
51952024-02-08T06:41:21.856Z 14 280-299 36 36 36 57 57 57 F F F
51962024-02-08T06:41:21.856Z 15 300-319 44 44 44 68 68 68 F F F
51972024-02-08T06:41:21.856Z 16 320-339 43 43 43 67 67 67 F F F
51982024-02-08T06:41:21.856Z 17 340-359 44 44 44 68 68 68 F F F
51992024-02-08T06:41:21.857Z 18 360-379 47 47 47 72 72 72 F F F
52002024-02-08T06:41:21.857Z 19 380-399 39 39 39 59 59 59 F F F
52012024-02-08T06:41:21.857Z 20 400-419 46 46 46 70 70 70 F F F
52022024-02-08T06:41:21.857Z 21 420-439 46 46 46 70 70 70 F F F
52032024-02-08T06:41:21.857Z 22 440-459 44 44 44 68 68 68 F F F
52042024-02-08T06:41:21.857Z 23 460-479 39 39 39 59 59 59 F F F
52052024-02-08T06:41:21.857Z 24 480-499 41 41 41 64 64 64 F F F
52062024-02-08T06:41:21.857Z 25 500-519 42 42 42 65 65 65 F F F
52072024-02-08T06:41:21.857Z 26 520-539 46 46 46 70 70 70 F F F
52082024-02-08T06:41:21.857Z 27 540-559 47 47 47 72 72 72 F F F
52092024-02-08T06:41:21.857Z 28 560-579 42 42 42 65 65 65 F F F
52102024-02-08T06:41:21.857Z 29 580-599 44 44 44 68 68 68 F F F
52112024-02-08T06:41:21.857ZMax gen: 47, Max flush: 72
52122024-02-08T06:41:21.857Z
52132024-02-08T06:41:21.857ZBegin loop 24
52142024-02-08T06:41:21.857ZDownstairs to restart: 1
52152024-02-08T06:41:21.879Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.119414121Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330}
52162024-02-08T06:41:21.885Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
52172024-02-08T06:41:21.885Z,"time":"2024-02-08T06:41:21.11969866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330}
52182024-02-08T06:41:21.885Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.119736567Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330}
52192024-02-08T06:41:21.885Z{"msg":"Crucible 87d6f44a-c14e-41f2-9d99-ecce7ff3fbf1 has session id: bba9997c-be7f-45b0-9503-8526e2263aa8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.119780544Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52202024-02-08T06:41:21.885Z{"msg":"Upstairs opts: Upstairs UUID: 87d6f44a-c14e-41f2-9d99-ecce7ff3fbf1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.119812751Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52212024-02-08T06:41:21.885Z{"msg":"Crucible stats registered with UUID: 87d6f44a-c14e-41f2-9d99-ecce7ff3fbf1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.119848458Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52222024-02-08T06:41:21.885Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.119882466Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330}
52232024-02-08T06:41:21.885Z{"msg":"87d6f44a-c14e-41f2-9d99-ecce7ff3fbf1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.119912964Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52242024-02-08T06:41:21.885Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.11995491Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52252024-02-08T06:41:21.885Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.119984608Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52262024-02-08T06:41:21.885Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.120010626Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52272024-02-08T06:41:21.885Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.120036615Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52282024-02-08T06:41:21.885Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.120075081Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52292024-02-08T06:41:21.885Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.12010116Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52302024-02-08T06:41:21.886Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.120124308Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"io task","client":"2","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52312024-02-08T06:41:21.886Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.120149716Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"io task","client":"1","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52322024-02-08T06:41:21.886Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.120167375Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"io task","client":"0","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52332024-02-08T06:41:21.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.120279386Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"io task","client":"2","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52342024-02-08T06:41:21.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.120474191Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"io task","client":"0","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52352024-02-08T06:41:21.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.120535177Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"io task","client":"1","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52362024-02-08T06:41:21.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.12089789Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52372024-02-08T06:41:21.886Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.120932277Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52382024-02-08T06:41:21.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.121101715Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52392024-02-08T06:41:21.886Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.12116196Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52402024-02-08T06:41:21.886Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.121490345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52412024-02-08T06:41:21.886Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.121630705Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52422024-02-08T06:41:21.886Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.12210006Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52432024-02-08T06:41:21.886Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.122162595Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52442024-02-08T06:41:21.886Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.122216381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52452024-02-08T06:41:21.886Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.122237279Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52462024-02-08T06:41:21.887Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.122261817Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52472024-02-08T06:41:21.887Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.122277416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52482024-02-08T06:41:21.887Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.122483321Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52492024-02-08T06:41:21.887Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.122521118Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52502024-02-08T06:41:21.887Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123038299Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52512024-02-08T06:41:21.887Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123477666Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52522024-02-08T06:41:21.887Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123522673Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52532024-02-08T06:41:21.887Z{"msg":"[0]R flush_numbers[0..12]: [62, 68, 65, 71, 71, 71, 57, 70, 72, 66, 68, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.12355881Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52542024-02-08T06:41:21.887Z{"msg":"[0]R generation[0..12]: [40, 44, 42, 46, 46, 46, 36, 46, 47, 42, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123595357Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52552024-02-08T06:41:21.887Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123630035Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52562024-02-08T06:41:21.887Z{"msg":"[1]R flush_numbers[0..12]: [62, 68, 65, 71, 71, 71, 57, 70, 72, 66, 68, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123665702Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52572024-02-08T06:41:21.887Z{"msg":"[1]R generation[0..12]: [40, 44, 42, 46, 46, 46, 36, 46, 47, 42, 44, 46]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
52582024-02-08T06:41:21.887Z,"level":30,"time":"2024-02-08T06:41:21.123725918Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52592024-02-08T06:41:21.887Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123768224Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52602024-02-08T06:41:21.887Z{"msg":"[2]R flush_numbers[0..12]: [62, 68, 65, 71, 71, 71, 57, 70, 72, 66, 68, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123807851Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52612024-02-08T06:41:21.888Z{"msg":"[2]R generation[0..12]: [40, 44, 42, 46, 46, 46, 36, 46, 47, 42, 44, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123844789Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52622024-02-08T06:41:21.888Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123883626Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52632024-02-08T06:41:21.888Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123920883Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52642024-02-08T06:41:21.888Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.12395723Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52652024-02-08T06:41:21.888Z{"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.123987278Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52662024-02-08T06:41:21.888Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.124005377Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52672024-02-08T06:41:21.888Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.124044763Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52682024-02-08T06:41:21.888Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.124059962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52692024-02-08T06:41:21.888Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.124079111Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52702024-02-08T06:41:21.888Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.124110119Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52712024-02-08T06:41:21.888Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.124130097Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52722024-02-08T06:41:21.888Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.124156925Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52732024-02-08T06:41:21.889Z{"msg":"87d6f44a-c14e-41f2-9d99-ecce7ff3fbf1 is now active with session: bba9997c-be7f-45b0-9503-8526e2263aa8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.124175784Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52742024-02-08T06:41:21.889Zsession_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52752024-02-08T06:41:21.889Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.12423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330,"session_id":"bba9997c-be7f-45b0-9503-8526e2263aa8"}
52762024-02-08T06:41:21.889Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:21.124263917Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330}
52772024-02-08T06:41:21.889ZLoading write count information from file "/tmp/test_repair_verify.data"
52782024-02-08T06:41:21.889ZRun Repair workload
52792024-02-08T06:41:21.889Z01/30 Write block 531 len 32768 data: 5 4 5 5 4 5 6 5
52802024-02-08T06:41:21.889Z02/30 Read block 466 len 20480
52812024-02-08T06:41:21.893Z03/30 Write block 467 len 36864 data: 3 3 5 5 4 6 5 5 5
52822024-02-08T06:41:21.893Z04/30 Write block 128 len 24576 data: 3 3 4 6 5 5
52832024-02-08T06:41:21.893Z05/30 Write block 243 len 16384 data: 8 7 7 7
52842024-02-08T06:41:21.893Z06/30 Read block 506 len 24576
52852024-02-08T06:41:21.893Z07/30 Read block 448 len 20480
52862024-02-08T06:41:21.893Z08/30 Flush
52872024-02-08T06:41:21.894Z09/30 Write block 229 len 24576 data: 10 8 5 5 5 5
52882024-02-08T06:41:21.894Z10/30 Write block 532 len 20480 data: 5 6 6 5 6
52892024-02-08T06:41:21.894Z11/30 Write block 461 len 12288 data: 4 5 5
52902024-02-08T06:41:21.894Z12/30 Read block 201 len 40960
52912024-02-08T06:41:21.894Z13/30 Read block 404 len 4096
52922024-02-08T06:41:21.897Z14/30 Read block 310 len 28672
52932024-02-08T06:41:21.897Z15/30 Write block 523 len 20480 data: 5 4 5 5 5
52942024-02-08T06:41:21.897Z16/30 Read block 439 len 40960
52952024-02-08T06:41:21.897Z17/30 Read block 10 len 40960
52962024-02-08T06:41:21.900Z18/30 Write block 169 len 28672 data: 5 4 4 4 5 3 4
52972024-02-08T06:41:21.900Z19/30 Read block 481 len 12288
52982024-02-08T06:41:21.900Z20/30 Read block 420 len 20480
52992024-02-08T06:41:21.901Z21/30 Write block 447 len 28672 data: 5 5 5 4 4 4 4
53002024-02-08T06:41:21.903Z22/30 Read block 325 len 20480
53012024-02-08T06:41:21.903Z23/30 Read block 473 len 8192
53022024-02-08T06:41:21.904Z24/30 Flush
53032024-02-08T06:41:21.906Z25/30 Write block 217 len 40960 data: 4 4 7 8 9 9 9 10 9 9
53042024-02-08T06:41:21.909Z26/30 Read block 590 len 16384
53052024-02-08T06:41:21.910Z27/30 Write block 467 len 36864 data: 4 4 6 6 5 7 6 6 6
53062024-02-08T06:41:21.913Z28/30 Write block 54 len 40960 data: 6 6 8 7 3 3 3 3 3 3
53072024-02-08T06:41:21.913Z29/30 Read block 396 len 24576
53082024-02-08T06:41:21.913Z30/30 Write block 529 len 32768 data: 6 7 6 6 7 7 6 7
53092024-02-08T06:41:21.913Z----------------------------------------------------------------
53102024-02-08T06:41:21.913Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
53112024-02-08T06:41:21.913ZStates: Active Active Active
53122024-02-08T06:41:21.914ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
53132024-02-08T06:41:21.914Z 1 Acked 1000 Write 8 Done Done Done false
53142024-02-08T06:41:21.914Z 2 Acked 1001 Read 5 Done Done Done false
53152024-02-08T06:41:21.914Z 3 Acked 1002 Write 9 Done Sent Done false
53162024-02-08T06:41:21.914Z 4 Acked 1003 Write 6 Done Sent Done false
53172024-02-08T06:41:21.914Z 5 Acked 1004 Write 4 Done Sent Done false
53182024-02-08T06:41:21.914Z 6 Acked 1005 Read 6 Done Sent Done false
53192024-02-08T06:41:21.914Z 7 Acked 1006 Read 5 Done Sent Done false
53202024-02-08T06:41:21.914Z 8 Acked 1007 Flush 0 Done Sent Done false
53212024-02-08T06:41:21.914Z 9 Acked 1008 Write 6 Done Sent Done false
53222024-02-08T06:41:21.914Z 10 Acked 1009 Write 5 Done Sent Done false
53232024-02-08T06:41:21.914Z 11 Acked 1010 Write 3 Done Sent Done false
53242024-02-08T06:41:21.914Z 12 Acked 1011 Read 10 Done Sent Done false
53252024-02-08T06:41:21.914Z 13 Acked 1012 Read 1 Done Sent Done false
53262024-02-08T06:41:21.914Z 14 Acked 1013 Read 7 Done Sent Done false
53272024-02-08T06:41:21.915Z 15 Acked 1014 Write 5 Done Sent Done false
53282024-02-08T06:41:21.915Z 16 Acked 1015 Read 10 Done Sent Done false
53292024-02-08T06:41:21.915Z 17 Acked 1016 Read 10 Done Sent Done false
53302024-02-08T06:41:21.915Z 18 Acked 1017 Write 7 Done Sent Done false
53312024-02-08T06:41:21.915Z 19 Acked 1018 Read 3 Done Sent Done false
53322024-02-08T06:41:21.915Z 20 Acked 1019 Read 5 Done Sent Done false
53332024-02-08T06:41:21.915Z 21 Acked 1020 Write 7 Done Sent Done false
53342024-02-08T06:41:21.915Z 22 Acked 1021 Read 5 Done Sent Done false
53352024-02-08T06:41:21.915Z 23 Acked 1022 Read 2 Done Sent Done false
53362024-02-08T06:41:21.915Z 24 Acked 1023 Flush 0 Done Sent Done false
53372024-02-08T06:41:21.915Z 25 Acked 1024 Write 10 Done Sent Done false
53382024-02-08T06:41:21.915Z 26 Acked 1025 Read 4 Done Sent Done false
53392024-02-08T06:41:21.915Z 27 Acked 1026 Write 9 Done Sent Done false
53402024-02-08T06:41:21.915Z 28 Acked 1027 Write 10 Done Sent Done false
53412024-02-08T06:41:21.915Z 29 Acked 1028 Read 6 Done Sent Done false
53422024-02-08T06:41:21.915Z 30 Acked 1029 Write 8 Sent Sent Sent false
53432024-02-08T06:41:21.916Z STATES DS:0 DS:1 DS:2 TOTAL
53442024-02-08T06:41:21.916Z New 0 0 0 0
53452024-02-08T06:41:21.916Z Sent 1 28 1 30
53462024-02-08T06:41:21.916Z Done 29 2 29 60
53472024-02-08T06:41:21.916Z Skipped 0 0 0 0
53482024-02-08T06:41:21.916Z Error 0 0 0 0
53492024-02-08T06:41:21.916ZLast Flush: 1023 0 1023
53502024-02-08T06:41:21.916ZDownstairs last five completed:
53512024-02-08T06:41:21.916ZUpstairs last five completed: 30 29 28 27 26
53522024-02-08T06:41:21.916Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:21.157489153Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1330}
53532024-02-08T06:41:21.916ZWrote out file "/tmp/test_repair_verify.data"
53542024-02-08T06:41:21.919Z
53552024-02-08T06:41:21.922ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53562024-02-08T06:41:21.937ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53572024-02-08T06:41:21.937Z 0 000-019 40 40 40 62 62 62 F F F
53582024-02-08T06:41:21.937Z 1 020-039 44 44 44 68 68 68 F F F
53592024-02-08T06:41:21.937Z 2 040-059 42 42 42 65 65 65 T F T <---
53602024-02-08T06:41:21.937Z 3 060-079 46 46 46 71 71 71 T F T <---
53612024-02-08T06:41:21.937Z 4 080-099 46 46 46 71 71 71 F F F
53622024-02-08T06:41:21.941Z 5 100-119 46 46 46 71 71 71 F F F
53632024-02-08T06:41:21.941Z 6 120-139 48 36 48 73 57 73 F F F <---
53642024-02-08T06:41:21.942Z 7 140-159 46 46 46 70 70 70 F F F
53652024-02-08T06:41:21.942Z 8 160-179 48 47 48 74 72 74 F F F <---
53662024-02-08T06:41:21.942Z 9 180-199 42 42 42 66 66 66 F F F
53672024-02-08T06:41:21.942Z 10 200-219 44 44 44 68 68 68 T F T <---
53682024-02-08T06:41:21.942Z 11 220-239 48 46 48 74 71 74 T F T <---
53692024-02-08T06:41:21.942Z 12 240-259 48 46 48 73 71 73 F F F <---
53702024-02-08T06:41:21.942Z 13 260-279 46 46 46 70 70 70 F F F
53712024-02-08T06:41:21.942Z 14 280-299 36 36 36 57 57 57 F F F
53722024-02-08T06:41:21.942Z 15 300-319 44 44 44 68 68 68 F F F
53732024-02-08T06:41:21.942Z 16 320-339 43 43 43 67 67 67 F F F
53742024-02-08T06:41:21.942Z 17 340-359 44 44 44 68 68 68 F F F
53752024-02-08T06:41:21.942Z 18 360-379 47 47 47 72 72 72 F F F
53762024-02-08T06:41:21.942Z 19 380-399 39 39 39 59 59 59 F F F
53772024-02-08T06:41:21.942Z 20 400-419 46 46 46 70 70 70 F F F
53782024-02-08T06:41:21.943Z 21 420-439 46 46 46 70 70 70 F F F
53792024-02-08T06:41:21.943Z 22 440-459 48 44 48 74 68 74 F F F <---
53802024-02-08T06:41:21.943Z 23 460-479 48 39 48 74 59 74 T F T <---
53812024-02-08T06:41:21.943Z 24 480-499 41 41 41 64 64 64 F F F
53822024-02-08T06:41:21.943Z 25 500-519 42 42 42 65 65 65 F F F
53832024-02-08T06:41:21.943Z 26 520-539 48 46 48 74 70 74 T T T <---
53842024-02-08T06:41:21.943Z 27 540-559 47 47 47 72 72 72 F F F
53852024-02-08T06:41:21.943Z 28 560-579 42 42 42 65 65 65 F F F
53862024-02-08T06:41:21.943Z 29 580-599 44 44 44 68 68 68 F F F
53872024-02-08T06:41:21.943ZMax gen: 48, Max flush: 74
53882024-02-08T06:41:21.943ZError: Difference in extent metadata found!
53892024-02-08T06:41:21.943ZA Difference in extent metadata is expected here
53902024-02-08T06:41:21.943ZOn loop 24
53912024-02-08T06:41:21.943Z
53922024-02-08T06:41:21.947ZVerifying data now
53932024-02-08T06:41:27.467ZNo change after verify
53942024-02-08T06:41:27.467ZLoop: 24 Downstairs dump after verify (and repair):
53952024-02-08T06:41:27.472ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53962024-02-08T06:41:27.472Z 0 000-019 40 40 40 62 62 62 F F F
53972024-02-08T06:41:27.473Z 1 020-039 44 44 44 68 68 68 F F F
53982024-02-08T06:41:27.473Z 2 040-059 49 49 49 75 75 75 F F F
53992024-02-08T06:41:27.473Z 3 060-079 49 49 49 75 75 75 F F F
54002024-02-08T06:41:27.473Z 4 080-099 46 46 46 71 71 71 F F F
54012024-02-08T06:41:27.473Z 5 100-119 46 46 46 71 71 71 F F F
54022024-02-08T06:41:27.473Z 6 120-139 48 48 48 73 73 73 F F F
54032024-02-08T06:41:27.476Z 7 140-159 46 46 46 70 70 70 F F F
54042024-02-08T06:41:27.476Z 8 160-179 48 48 48 74 74 74 F F F
54052024-02-08T06:41:27.476Z 9 180-199 42 42 42 66 66 66 F F F
54062024-02-08T06:41:27.476Z 10 200-219 49 49 49 75 75 75 F F F
54072024-02-08T06:41:27.476Z 11 220-239 49 49 49 75 75 75 F F F
54082024-02-08T06:41:27.476Z 12 240-259 48 48 48 73 73 73 F F F
54092024-02-08T06:41:27.476Z 13 260-279 46 46 46 70 70 70 F F F
54102024-02-08T06:41:27.476Z 14 280-299 36 36 36 57 57 57 F F F
54112024-02-08T06:41:27.476Z 15 300-319 44 44 44 68 68 68 F F F
54122024-02-08T06:41:27.477Z 16 320-339 43 43 43 67 67 67 F F F
54132024-02-08T06:41:27.477Z 17 340-359 44 44 44 68 68 68 F F F
54142024-02-08T06:41:27.477Z 18 360-379 47 47 47 72 72 72 F F F
54152024-02-08T06:41:27.477Z 19 380-399 39 39 39 59 59 59 F F F
54162024-02-08T06:41:27.477Z 20 400-419 46 46 46 70 70 70 F F F
54172024-02-08T06:41:27.477Z 21 420-439 46 46 46 70 70 70 F F F
54182024-02-08T06:41:27.477Z 22 440-459 48 48 48 74 74 74 F F F
54192024-02-08T06:41:27.477Z 23 460-479 49 49 49 75 75 75 F F F
54202024-02-08T06:41:27.477Z 24 480-499 41 41 41 64 64 64 F F F
54212024-02-08T06:41:27.477Z 25 500-519 42 42 42 65 65 65 F F F
54222024-02-08T06:41:27.477Z 26 520-539 49 49 49 75 75 75 F F F
54232024-02-08T06:41:27.477Z 27 540-559 47 47 47 72 72 72 F F F
54242024-02-08T06:41:27.477Z 28 560-579 42 42 42 65 65 65 F F F
54252024-02-08T06:41:27.477Z 29 580-599 44 44 44 68 68 68 F F F
54262024-02-08T06:41:27.477ZMax gen: 49, Max flush: 75
54272024-02-08T06:41:27.477Z
54282024-02-08T06:41:27.478ZBegin loop 25
54292024-02-08T06:41:27.478ZDownstairs to restart: 0
54302024-02-08T06:41:27.494Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.738622083Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338}
54312024-02-08T06:41:27.494Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.738867285Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338}
54322024-02-08T06:41:27.499Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.738900682Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338}
54332024-02-08T06:41:27.499Z{"msg":"Crucible 8ba1d1e5-2928-4900-bf27-af0cd20c1b2e has session id: c9fbfa40-4251-4bf5-85e7-634163852330","v":0,"name":"crucible","level":30Crucible runtime is spawned
54342024-02-08T06:41:27.499Z,"time":"2024-02-08T06:41:26.73893449Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54352024-02-08T06:41:27.499Z{"msg":"Upstairs opts: Upstairs UUID: 8ba1d1e5-2928-4900-bf27-af0cd20c1b2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.738987506Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54362024-02-08T06:41:27.499Z{"msg":"Crucible stats registered with UUID: 8ba1d1e5-2928-4900-bf27-af0cd20c1b2e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739013244Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54372024-02-08T06:41:27.499Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739035332Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338}
54382024-02-08T06:41:27.499Z{"msg":"8ba1d1e5-2928-4900-bf27-af0cd20c1b2e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739133165Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54392024-02-08T06:41:27.499Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739181991Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54402024-02-08T06:41:27.499Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739235247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54412024-02-08T06:41:27.500Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739275294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54422024-02-08T06:41:27.500Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739318751Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54432024-02-08T06:41:27.500Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739394065Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54442024-02-08T06:41:27.500Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739445281Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54452024-02-08T06:41:27.500Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739473219Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"io task","client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54462024-02-08T06:41:27.500Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739500437Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"io task","client":"1","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54472024-02-08T06:41:27.500Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739536694Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"io task","client":"2","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54482024-02-08T06:41:27.500Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739578011Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"io task","client":"2","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54492024-02-08T06:41:27.500Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.7395991Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"io task","client":"1","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54502024-02-08T06:41:27.500Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:26.73972617Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"io task","client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54512024-02-08T06:41:27.500Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:26.739788696Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"io task","client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54522024-02-08T06:41:27.500Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:26.739874349Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54532024-02-08T06:41:27.500Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739902537Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54542024-02-08T06:41:27.500Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739940244Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54552024-02-08T06:41:27.500Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.739965062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"io task","client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54562024-02-08T06:41:27.501Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.740298677Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54572024-02-08T06:41:27.501Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.740343404Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54582024-02-08T06:41:27.501Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.740372732Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54592024-02-08T06:41:27.501Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.740410499Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54602024-02-08T06:41:27.501Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.740875974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54612024-02-08T06:41:27.501Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.741024263Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54622024-02-08T06:41:27.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.741427462Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54632024-02-08T06:41:27.501Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.74146261Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54642024-02-08T06:41:27.501Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.741511836Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54652024-02-08T06:41:27.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.741566322Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54662024-02-08T06:41:27.501Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.741603129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54672024-02-08T06:41:27.501Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:26.741619628Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54682024-02-08T06:41:28.498Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.741314513Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"io task","client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54692024-02-08T06:41:28.505Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.741512078Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"io task","client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54702024-02-08T06:41:28.505Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.742539291Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54712024-02-08T06:41:28.505Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.742573819Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54722024-02-08T06:41:28.505Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743067772Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54732024-02-08T06:41:28.505Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743631619Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54742024-02-08T06:41:28.505Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743670976Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54752024-02-08T06:41:28.505Z{"msg":"[0]R flush_numbers[0..12]: [62, 68, 75, 75, 71, 71, 73, 70, 74, 66, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743686775Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54762024-02-08T06:41:28.505Z{"msg":"[0]R generation[0..12]: [40, 44, 49, 49, 46, 46, 48, 46, 48, 42, 49, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743744581Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54772024-02-08T06:41:28.506Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743770769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54782024-02-08T06:41:28.506Z{"msg":"[1]R flush_numbers[0..12]: [62, 68, 75, 75, 71, 71, 73, 70, 74, 66, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743805106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54792024-02-08T06:41:28.506Z{"msg":"[1]R generation[0..12]: [40, 44, 49, 49, 46, 46, 48, 46, 48, 42, 49, 49]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
54802024-02-08T06:41:28.506Z,"time":"2024-02-08T06:41:27.743826625Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54812024-02-08T06:41:28.506Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743845573Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54822024-02-08T06:41:28.506Z{"msg":"[2]R flush_numbers[0..12]: [62, 68, 75, 75, 71, 71, 73, 70, 74, 66, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743860672Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54832024-02-08T06:41:28.506Z{"msg":"[2]R generation[0..12]: [40, 44, 49, 49, 46, 46, 48, 46, 48, 42, 49, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743874771Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54842024-02-08T06:41:28.506Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.74388884Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54852024-02-08T06:41:28.506Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743909158Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54862024-02-08T06:41:28.506Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743923208Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54872024-02-08T06:41:28.506Z{"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743937056Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54882024-02-08T06:41:28.506Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743956895Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54892024-02-08T06:41:28.506Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743970594Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54902024-02-08T06:41:28.506Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.743983663Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54912024-02-08T06:41:28.506Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.744003991Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54922024-02-08T06:41:28.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.74401749Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54932024-02-08T06:41:28.507Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.744056758Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54942024-02-08T06:41:28.507Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.744069906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54952024-02-08T06:41:28.507Z{"msg":"8ba1d1e5-2928-4900-bf27-af0cd20c1b2e is now active with session: c9fbfa40-4251-4bf5-85e7-634163852330","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.744083495Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54962024-02-08T06:41:28.507Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.744096064Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338,"session_id":"c9fbfa40-4251-4bf5-85e7-634163852330"}
54972024-02-08T06:41:28.507Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:27.744108634Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338}
54982024-02-08T06:41:28.507ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54992024-02-08T06:41:28.507ZLoading write count information from file "/tmp/test_repair_verify.data"
55002024-02-08T06:41:28.507ZRun Repair workload
55012024-02-08T06:41:28.507Z01/30 Write block 298 len 40960 data: 6 9 7 6 6 5 5 5 4 3
55022024-02-08T06:41:28.511Z02/30 Read block 551 len 32768
55032024-02-08T06:41:28.511Z03/30 Write block 172 len 24576 data: 5 6 4 5 5 6
55042024-02-08T06:41:28.511Z04/30 Write block 212 len 24576 data: 6 6 6 4 4 5
55052024-02-08T06:41:28.511Z05/30 Write block 549 len 12288 data: 6 4 5
55062024-02-08T06:41:28.511Z06/30 Read block 30 len 8192
55072024-02-08T06:41:28.511Z07/30 Read block 571 len 12288
55082024-02-08T06:41:28.511Z08/30 Write block 590 len 8192 data: 6 5
55092024-02-08T06:41:28.511Z09/30 Flush
55102024-02-08T06:41:28.511Z10/30 Write block 318 len 32768 data: 5 7 7 5 5 5 3 4
55112024-02-08T06:41:28.514Z11/30 Read block 346 len 32768
55122024-02-08T06:41:28.514Z12/30 Flush
55132024-02-08T06:41:28.514Z13/30 Write block 108 len 4096 data: 6
55142024-02-08T06:41:28.517Z14/30 Read block 267 len 16384
55152024-02-08T06:41:28.517Z15/30 Read block 177 len 20480
55162024-02-08T06:41:28.517Z16/30 Read block 219 len 12288
55172024-02-08T06:41:28.517Z17/30 Read block 581 len 12288
55182024-02-08T06:41:28.520Z18/30 Read block 65 len 16384
55192024-02-08T06:41:28.520Z19/30 Read block 457 len 8192
55202024-02-08T06:41:28.521Z20/30 Write block 170 len 32768 data: 5 5 6 7 5 6 6 7
55212024-02-08T06:41:28.521Z21/30 Read block 500 len 20480
55222024-02-08T06:41:28.523Z22/30 Read block 530 len 28672
55232024-02-08T06:41:28.523Z23/30 Read block 309 len 16384
55242024-02-08T06:41:28.524Z24/30 Write block 148 len 8192 data: 3 5
55252024-02-08T06:41:28.524Z25/30 Read block 325 len 16384
55262024-02-08T06:41:28.527Z26/30 Read block 63 len 28672
55272024-02-08T06:41:28.527Z27/30 Read block 289 len 28672
55282024-02-08T06:41:28.527Z28/30 Write block 443 len 32768 data: 5 4 4 5 6 6 6 5
55292024-02-08T06:41:28.530Z29/30 Read block 165 len 40960
55302024-02-08T06:41:28.530Z30/30 Read block 102 len 40960
55312024-02-08T06:41:28.530Z----------------------------------------------------------------
55322024-02-08T06:41:28.530Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
55332024-02-08T06:41:28.530ZStates: Active Active Active
55342024-02-08T06:41:28.531ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
55352024-02-08T06:41:28.531Z 1 Acked 1000 Write 10 Done Done Done false
55362024-02-08T06:41:28.531Z 2 Acked 1001 Read 8 Done Done Done false
55372024-02-08T06:41:28.531Z 3 Acked 1002 Write 6 Sent Done Done false
55382024-02-08T06:41:28.531Z 4 Acked 1003 Write 6 Sent Done Done false
55392024-02-08T06:41:28.531Z 5 Acked 1004 Write 3 Sent Done Done false
55402024-02-08T06:41:28.531Z 6 Acked 1005 Read 2 Sent Done Done false
55412024-02-08T06:41:28.531Z 7 Acked 1006 Read 3 Sent Done Done false
55422024-02-08T06:41:28.531Z 8 Acked 1007 Write 2 Sent Done Done false
55432024-02-08T06:41:28.531Z 9 Acked 1008 Flush 0 Sent Done Done false
55442024-02-08T06:41:28.531Z 10 Acked 1009 Write 8 Sent Done Done false
55452024-02-08T06:41:28.531Z 11 Acked 1010 Read 8 Sent Done Done false
55462024-02-08T06:41:28.531Z 12 Acked 1011 Flush 0 Sent Done Done false
55472024-02-08T06:41:28.531Z 13 Acked 1012 Write 1 Sent Done Done false
55482024-02-08T06:41:28.532Z 14 Acked 1013 Read 4 Sent Done Done false
55492024-02-08T06:41:28.532Z 15 Acked 1014 Read 5 Sent Done Done false
55502024-02-08T06:41:28.532Z 16 Acked 1015 Read 3 Sent Done Done false
55512024-02-08T06:41:28.532Z 17 Acked 1016 Read 3 Sent Done Done false
55522024-02-08T06:41:28.532Z 18 Acked 1017 Read 4 Sent Done Done false
55532024-02-08T06:41:28.532Z 19 Acked 1018 Read 2 Sent Done Done false
55542024-02-08T06:41:28.532Z 20 Acked 1019 Write 8 Sent Done Done false
55552024-02-08T06:41:28.532Z 21 Acked 1020 Read 5 Sent Done Done false
55562024-02-08T06:41:28.532Z 22 Acked 1021 Read 7 Sent Done Done false
55572024-02-08T06:41:28.532Z 23 Acked 1022 Read 4 Sent Done Done false
55582024-02-08T06:41:28.532Z 24 Acked 1023 Write 2 Sent Done Done false
55592024-02-08T06:41:28.532Z 25 Acked 1024 Read 4 Sent Done Done false
55602024-02-08T06:41:28.532Z 26 Acked 1025 Read 7 Sent Done Done false
55612024-02-08T06:41:28.532Z 27 Acked 1026 Read 7 Sent Done Done false
55622024-02-08T06:41:28.532Z 28 Acked 1027 Write 8 Sent Done Done false
55632024-02-08T06:41:28.533Z 29 Acked 1028 Read 10 Sent Done Done false
55642024-02-08T06:41:28.533Z 30 Acked 1029 Read 10 Sent Sent Done false
55652024-02-08T06:41:28.533Z STATES DS:0 DS:1 DS:2 TOTAL
55662024-02-08T06:41:28.533Z New 0 0 0 0
55672024-02-08T06:41:28.533Z Sent 28 1 0 29
55682024-02-08T06:41:28.533Z Done 2 29 30 61
55692024-02-08T06:41:28.533Z Skipped 0 0 0 0
55702024-02-08T06:41:28.533Z Error 0 0 0 0
55712024-02-08T06:41:28.533ZLast Flush: 0 1011 1011
55722024-02-08T06:41:28.533ZDownstairs last five completed:
55732024-02-08T06:41:28.533ZUpstairs last five completed: 30 29 28 27 26
55742024-02-08T06:41:28.533Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:27.77399428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1338}
55752024-02-08T06:41:28.533ZWrote out file "/tmp/test_repair_verify.data"
55762024-02-08T06:41:28.536Z
55772024-02-08T06:41:28.539ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55782024-02-08T06:41:28.555ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55792024-02-08T06:41:28.555Z 0 000-019 40 40 40 62 62 62 F F F
55802024-02-08T06:41:28.555Z 1 020-039 44 44 44 68 68 68 F F F
55812024-02-08T06:41:28.555Z 2 040-059 49 49 49 75 75 75 F F F
55822024-02-08T06:41:28.555Z 3 060-079 49 49 49 75 75 75 F F F
55832024-02-08T06:41:28.555Z 4 080-099 46 46 46 71 71 71 F F F
55842024-02-08T06:41:28.555Z 5 100-119 46 46 46 71 71 71 F T T <---
55852024-02-08T06:41:28.555Z 6 120-139 48 48 48 73 73 73 F F F
55862024-02-08T06:41:28.555Z 7 140-159 46 46 46 70 70 70 F T T <---
55872024-02-08T06:41:28.555Z 8 160-179 48 50 50 74 76 76 F T T <---
55882024-02-08T06:41:28.555Z 9 180-199 42 42 42 66 66 66 F F F
55892024-02-08T06:41:28.556Z 10 200-219 49 50 50 75 76 76 F F F <---
55902024-02-08T06:41:28.559Z 11 220-239 49 49 49 75 75 75 F F F
55912024-02-08T06:41:28.559Z 12 240-259 48 48 48 73 73 73 F F F
55922024-02-08T06:41:28.559Z 13 260-279 46 46 46 70 70 70 F F F
55932024-02-08T06:41:28.559Z 14 280-299 36 50 50 57 76 76 T F F <---
55942024-02-08T06:41:28.559Z 15 300-319 44 50 50 68 77 77 T F F <---
55952024-02-08T06:41:28.559Z 16 320-339 43 50 50 67 77 77 F F F <---
55962024-02-08T06:41:28.559Z 17 340-359 44 44 44 68 68 68 F F F
55972024-02-08T06:41:28.559Z 18 360-379 47 47 47 72 72 72 F F F
55982024-02-08T06:41:28.559Z 19 380-399 39 39 39 59 59 59 F F F
55992024-02-08T06:41:28.559Z 20 400-419 46 46 46 70 70 70 F F F
56002024-02-08T06:41:28.559Z 21 420-439 46 46 46 70 70 70 F F F
56012024-02-08T06:41:28.559Z 22 440-459 48 48 48 74 74 74 F T T <---
56022024-02-08T06:41:28.559Z 23 460-479 49 49 49 75 75 75 F F F
56032024-02-08T06:41:28.559Z 24 480-499 41 41 41 64 64 64 F F F
56042024-02-08T06:41:28.559Z 25 500-519 42 42 42 65 65 65 F F F
56052024-02-08T06:41:28.560Z 26 520-539 49 49 49 75 75 75 F F F
56062024-02-08T06:41:28.560Z 27 540-559 47 50 50 72 76 76 F F F <---
56072024-02-08T06:41:28.560Z 28 560-579 42 42 42 65 65 65 F F F
56082024-02-08T06:41:28.560Z 29 580-599 44 50 50 68 76 76 F F F <---
56092024-02-08T06:41:28.560ZMax gen: 50, Max flush: 77
56102024-02-08T06:41:28.560ZError: Difference in extent metadata found!
56112024-02-08T06:41:28.560ZA Difference in extent metadata is expected here
56122024-02-08T06:41:28.560ZOn loop 25
56132024-02-08T06:41:28.560Z
56142024-02-08T06:41:28.563ZVerifying data now
56152024-02-08T06:41:34.073ZNo change after verify
56162024-02-08T06:41:34.073ZLoop: 25 Downstairs dump after verify (and repair):
56172024-02-08T06:41:34.089ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
56182024-02-08T06:41:34.089Z 0 000-019 40 40 40 62 62 62 F F F
56192024-02-08T06:41:34.089Z 1 020-039 44 44 44 68 68 68 F F F
56202024-02-08T06:41:34.089Z 2 040-059 49 49 49 75 75 75 F F F
56212024-02-08T06:41:34.089Z 3 060-079 49 49 49 75 75 75 F F F
56222024-02-08T06:41:34.089Z 4 080-099 46 46 46 71 71 71 F F F
56232024-02-08T06:41:34.089Z 5 100-119 51 51 51 78 78 78 F F F
56242024-02-08T06:41:34.089Z 6 120-139 48 48 48 73 73 73 F F F
56252024-02-08T06:41:34.089Z 7 140-159 51 51 51 78 78 78 F F F
56262024-02-08T06:41:34.089Z 8 160-179 51 51 51 78 78 78 F F F
56272024-02-08T06:41:34.089Z 9 180-199 42 42 42 66 66 66 F F F
56282024-02-08T06:41:34.089Z 10 200-219 50 50 50 76 76 76 F F F
56292024-02-08T06:41:34.089Z 11 220-239 49 49 49 75 75 75 F F F
56302024-02-08T06:41:34.089Z 12 240-259 48 48 48 73 73 73 F F F
56312024-02-08T06:41:34.093Z 13 260-279 46 46 46 70 70 70 F F F
56322024-02-08T06:41:34.093Z 14 280-299 50 50 50 76 76 76 F F F
56332024-02-08T06:41:34.093Z 15 300-319 50 50 50 77 77 77 F F F
56342024-02-08T06:41:34.093Z 16 320-339 50 50 50 77 77 77 F F F
56352024-02-08T06:41:34.093Z 17 340-359 44 44 44 68 68 68 F F F
56362024-02-08T06:41:34.093Z 18 360-379 47 47 47 72 72 72 F F F
56372024-02-08T06:41:34.093Z 19 380-399 39 39 39 59 59 59 F F F
56382024-02-08T06:41:34.093Z 20 400-419 46 46 46 70 70 70 F F F
56392024-02-08T06:41:34.093Z 21 420-439 46 46 46 70 70 70 F F F
56402024-02-08T06:41:34.094Z 22 440-459 51 51 51 78 78 78 F F F
56412024-02-08T06:41:34.094Z 23 460-479 49 49 49 75 75 75 F F F
56422024-02-08T06:41:34.094Z 24 480-499 41 41 41 64 64 64 F F F
56432024-02-08T06:41:34.094Z 25 500-519 42 42 42 65 65 65 F F F
56442024-02-08T06:41:34.094Z 26 520-539 49 49 49 75 75 75 F F F
56452024-02-08T06:41:34.094Z 27 540-559 50 50 50 76 76 76 F F F
56462024-02-08T06:41:34.094Z 28 560-579 42 42 42 65 65 65 F F F
56472024-02-08T06:41:34.094Z 29 580-599 50 50 50 76 76 76 F F F
56482024-02-08T06:41:34.094ZMax gen: 51, Max flush: 78
56492024-02-08T06:41:34.094Z
56502024-02-08T06:41:34.094ZBegin loop 26
56512024-02-08T06:41:34.094ZDownstairs to restart: 2
56522024-02-08T06:41:34.114Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.354808374Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346}
56532024-02-08T06:41:34.119Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355079993Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346}
56542024-02-08T06:41:34.119Z{Crucible runtime is spawned
56552024-02-08T06:41:34.119Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.35512584Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346}
56562024-02-08T06:41:34.119Z{"msg":"Crucible d72f4c64-4044-4ef8-a15d-25210799c8d2 has session id: a39d3ce8-c02b-4dd0-be2f-17e535b5147f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355162717Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56572024-02-08T06:41:34.119Z{"msg":"Upstairs opts: Upstairs UUID: d72f4c64-4044-4ef8-a15d-25210799c8d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355203384Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56582024-02-08T06:41:34.119Z{"msg":"Crucible stats registered with UUID: d72f4c64-4044-4ef8-a15d-25210799c8d2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355240211Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56592024-02-08T06:41:34.120Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.35525823Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346}
56602024-02-08T06:41:34.120Z{"msg":"d72f4c64-4044-4ef8-a15d-25210799c8d2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355279328Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56612024-02-08T06:41:34.120Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355293457Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56622024-02-08T06:41:34.120Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355325725Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56632024-02-08T06:41:34.120Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355358972Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56642024-02-08T06:41:34.120Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.35538389Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56652024-02-08T06:41:34.120Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355421868Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56662024-02-08T06:41:34.120Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355449205Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56672024-02-08T06:41:34.120Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355480393Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56682024-02-08T06:41:34.120Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355504941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"io task","client":"0","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56692024-02-08T06:41:34.120Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355540259Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"io task","client":"1","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56702024-02-08T06:41:34.120Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355699457Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"io task","client":"1","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56712024-02-08T06:41:34.120Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355755072Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"io task","client":"0","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56722024-02-08T06:41:34.120Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:33.355809938Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56732024-02-08T06:41:34.120Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:33.355871134Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56742024-02-08T06:41:34.121Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:33.35591755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56752024-02-08T06:41:34.121Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355950728Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56762024-02-08T06:41:34.121Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.355970916Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56772024-02-08T06:41:34.121Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.356116925Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56782024-02-08T06:41:34.121Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.356482488Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56792024-02-08T06:41:34.121Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.356529724Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56802024-02-08T06:41:34.121Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.356579941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56812024-02-08T06:41:34.121Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.356620338Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56822024-02-08T06:41:34.121Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.357107791Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56832024-02-08T06:41:34.121Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.357156597Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56842024-02-08T06:41:34.121Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.357665319Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56852024-02-08T06:41:34.121Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.357716615Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56862024-02-08T06:41:34.121Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.357755792Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56872024-02-08T06:41:34.121Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.35778401Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56882024-02-08T06:41:34.121Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.357814458Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56892024-02-08T06:41:34.121Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:33.357833596Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56902024-02-08T06:41:35.114Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.357646323Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56912024-02-08T06:41:35.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.357797741Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56922024-02-08T06:41:35.119Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.359044528Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56932024-02-08T06:41:35.119Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.359090154Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56942024-02-08T06:41:35.119Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.359565299Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56952024-02-08T06:41:35.119Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360092379Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56962024-02-08T06:41:35.119Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360121297Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56972024-02-08T06:41:35.119Z{"msg":"[0]R flush_numbers[0..12]: [62, 68, 75, 75, 71, 78, 73, 78, 78, 66, 76, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360136736Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56982024-02-08T06:41:35.119Z{"msg":"[0]R generation[0..12]: [40, 44, 49, 49, 46, 51, 48, 51, 51, 42, 50, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360151644Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
56992024-02-08T06:41:35.119Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360170903Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57002024-02-08T06:41:35.119Z{"msg":"[1]R flush_numbers[0..12]: [62, 68, 75, 75, 71, 78, 73, 78, 78, 66, 76, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360194091Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57012024-02-08T06:41:35.119Z{"msg":"[1]R generation[0..12]: [40, 44, 49, 49, 46, 51, 48, 51, 51, 42, 50, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.3602084Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57022024-02-08T06:41:35.120Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360223549Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57032024-02-08T06:41:35.120Z{"msg":"[2]R flush_numbers[0..12]: [62, 68, 75, 75, 71, 78, 73, 78, 78, 66, 76, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360245657Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57042024-02-08T06:41:35.120Z{"msg":"[2]R generation[0..12]: [40, 44, 49, 49, 46, 51, 48, 51, 51, 42, 50, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360260676Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57052024-02-08T06:41:35.120Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360274785Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57062024-02-08T06:41:35.120Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360295644Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57072024-02-08T06:41:35.120Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
57082024-02-08T06:41:35.120Z,"time":"2024-02-08T06:41:34.360310723Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57092024-02-08T06:41:35.120Z{"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360334421Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57102024-02-08T06:41:35.120Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.36034812Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57112024-02-08T06:41:35.120Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360369208Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57122024-02-08T06:41:35.120Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360382117Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57132024-02-08T06:41:35.120Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360396136Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57142024-02-08T06:41:35.120Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360416865Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57152024-02-08T06:41:35.120Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360430534Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57162024-02-08T06:41:35.120Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360443702Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57172024-02-08T06:41:35.120Z{"msg":"d72f4c64-4044-4ef8-a15d-25210799c8d2 is now active with session: a39d3ce8-c02b-4dd0-be2f-17e535b5147f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360463991Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57182024-02-08T06:41:35.121Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.36047729Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346,"session_id":"a39d3ce8-c02b-4dd0-be2f-17e535b5147f"}
57192024-02-08T06:41:35.121Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:34.360497429Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346}
57202024-02-08T06:41:35.121ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
57212024-02-08T06:41:35.121ZLoading write count information from file "/tmp/test_repair_verify.data"
57222024-02-08T06:41:35.121ZRun Repair workload
57232024-02-08T06:41:35.124Z01/30 Write block 540 len 20480 data: 6 5 4 4 4
57242024-02-08T06:41:35.124Z02/30 Write block 259 len 16384 data: 5 6 7 7
57252024-02-08T06:41:35.125Z03/30 Read block 419 len 32768
57262024-02-08T06:41:35.125Z04/30 Read block 397 len 24576
57272024-02-08T06:41:35.125Z05/30 Read block 569 len 12288
57282024-02-08T06:41:35.125Z06/30 Write block 474 len 36864 data: 7 7 3 2 2 2 3 3 4
57292024-02-08T06:41:35.125Z07/30 Flush
57302024-02-08T06:41:35.129Z08/30 Write block 384 len 12288 data: 6 5 4
57312024-02-08T06:41:35.129Z09/30 Read block 46 len 36864
57322024-02-08T06:41:35.129Z10/30 Read block 136 len 12288
57332024-02-08T06:41:35.132Z11/30 Read block 131 len 32768
57342024-02-08T06:41:35.132Z12/30 Flush
57352024-02-08T06:41:35.132Z13/30 Write block 300 len 28672 data: 8 7 7 6 6 6 5
57362024-02-08T06:41:35.133Z14/30 Write block 261 len 12288 data: 8 8 6
57372024-02-08T06:41:35.133Z15/30 Flush
57382024-02-08T06:41:35.136Z16/30 Write block 217 len 12288 data: 6 5 8
57392024-02-08T06:41:35.136Z17/30 Write block 501 len 8192 data: 3 3
57402024-02-08T06:41:35.142Z18/30 Read block 3 len 20480
57412024-02-08T06:41:35.142Z19/30 Read block 116 len 28672
57422024-02-08T06:41:35.142Z20/30 Read block 519 len 40960
57432024-02-08T06:41:35.145Z21/30 Write block 60 len 24576 data: 4 4 4 4 3 3
57442024-02-08T06:41:35.145Z22/30 Write block 351 len 36864 data: 8 6 5 6 6 7 8 7 6
57452024-02-08T06:41:35.145Z23/30 Read block 95 len 24576
57462024-02-08T06:41:35.145Z24/30 Read block 307 len 20480
57472024-02-08T06:41:35.145Z25/30 Flush
57482024-02-08T06:41:35.150Z26/30 Write block 570 len 40960 data: 5 4 4 3 3 3 3 3 3 4
57492024-02-08T06:41:35.150Z27/30 Read block 21 len 24576
57502024-02-08T06:41:35.150Z28/30 Write block 180 len 12288 data: 6 7 8
57512024-02-08T06:41:35.150Z29/30 Write block 437 len 8192 data: 7 5
57522024-02-08T06:41:35.150Z30/30 Write block 197 len 4096 data: 7
57532024-02-08T06:41:35.150Z----------------------------------------------------------------
57542024-02-08T06:41:35.151Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
57552024-02-08T06:41:35.151ZStates: Active Active Active
57562024-02-08T06:41:35.151ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57572024-02-08T06:41:35.151Z 1 Acked 1000 Write 5 Done Done Done false
57582024-02-08T06:41:35.151Z 2 Acked 1001 Write 4 Done Done Sent false
57592024-02-08T06:41:35.151Z 3 Acked 1002 Read 8 Done Done Sent false
57602024-02-08T06:41:35.151Z 4 Acked 1003 Read 6 Done Done Sent false
57612024-02-08T06:41:35.151Z 5 Acked 1004 Read 3 Done Done Sent false
57622024-02-08T06:41:35.151Z 6 Acked 1005 Write 9 Done Done Sent false
57632024-02-08T06:41:35.151Z 7 Acked 1006 Flush 0 Done Done Sent false
57642024-02-08T06:41:35.151Z 8 Acked 1007 Write 3 Done Done Sent false
57652024-02-08T06:41:35.151Z 9 Acked 1008 Read 9 Done Done Sent false
57662024-02-08T06:41:35.151Z 10 Acked 1009 Read 3 Done Done Sent false
57672024-02-08T06:41:35.151Z 11 Acked 1010 Read 8 Done Done Sent false
57682024-02-08T06:41:35.151Z 12 Acked 1011 Flush 0 Done Done Sent false
57692024-02-08T06:41:35.151Z 13 Acked 1012 Write 7 Done Done Sent false
57702024-02-08T06:41:35.152Z 14 Acked 1013 Write 3 Done Done Sent false
57712024-02-08T06:41:35.152Z 15 Acked 1014 Flush 0 Done Done Sent false
57722024-02-08T06:41:35.152Z 16 Acked 1015 Write 3 Done Done Sent false
57732024-02-08T06:41:35.152Z 17 Acked 1016 Write 2 Done Done Sent false
57742024-02-08T06:41:35.152Z 18 Acked 1017 Read 5 Done Done Sent false
57752024-02-08T06:41:35.152Z 19 Acked 1018 Read 7 Done Done Sent false
57762024-02-08T06:41:35.152Z 20 Acked 1019 Read 10 Done Done Sent false
57772024-02-08T06:41:35.152Z 21 Acked 1020 Write 6 Done Done Sent false
57782024-02-08T06:41:35.152Z 22 Acked 1021 Write 9 Done Done Sent false
57792024-02-08T06:41:35.152Z 23 Acked 1022 Read 6 Done Done Sent false
57802024-02-08T06:41:35.152Z 24 Acked 1023 Read 5 Done Done Sent false
57812024-02-08T06:41:35.152Z 25 Acked 1024 Flush 0 Done Done Sent false
57822024-02-08T06:41:35.152Z 26 Acked 1025 Write 10 Done Done Sent false
57832024-02-08T06:41:35.152Z 27 Acked 1026 Read 6 Done Done Sent false
57842024-02-08T06:41:35.153Z 28 Acked 1027 Write 3 Done Done Sent false
57852024-02-08T06:41:35.153Z 29 Acked 1028 Write 2 Done Done Sent false
57862024-02-08T06:41:35.153Z 30 Acked 1029 Write 1 Sent Sent Sent false
57872024-02-08T06:41:35.153Z STATES DS:0 DS:1 DS:2 TOTAL
57882024-02-08T06:41:35.153Z New 0 0 0 0
57892024-02-08T06:41:35.153Z Sent 1 1 29 31
57902024-02-08T06:41:35.153Z Done 29 29 1 59
57912024-02-08T06:41:35.153Z Skipped 0 0 0 0
57922024-02-08T06:41:35.153Z Error 0 0 0 0
57932024-02-08T06:41:35.153ZLast Flush: 1024 1024 0
57942024-02-08T06:41:35.153ZDownstairs last five completed:
57952024-02-08T06:41:35.153ZUpstairs last five completed: 30 29 28 27 26
57962024-02-08T06:41:35.153Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:34.393620222Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1346}
57972024-02-08T06:41:35.160ZWrote out file "/tmp/test_repair_verify.data"
57982024-02-08T06:41:35.160Z
57992024-02-08T06:41:35.160ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
58002024-02-08T06:41:35.175ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
58012024-02-08T06:41:35.175Z 0 000-019 40 40 40 62 62 62 F F F
58022024-02-08T06:41:35.175Z 1 020-039 44 44 44 68 68 68 F F F
58032024-02-08T06:41:35.175Z 2 040-059 49 49 49 75 75 75 F F F
58042024-02-08T06:41:35.175Z 3 060-079 52 52 49 82 82 75 F F F <---
58052024-02-08T06:41:35.175Z 4 080-099 46 46 46 71 71 71 F F F
58062024-02-08T06:41:35.175Z 5 100-119 51 51 51 78 78 78 F F F
58072024-02-08T06:41:35.175Z 6 120-139 48 48 48 73 73 73 F F F
58082024-02-08T06:41:35.175Z 7 140-159 51 51 51 78 78 78 F F F
58092024-02-08T06:41:35.175Z 8 160-179 51 51 51 78 78 78 F F F
58102024-02-08T06:41:35.175Z 9 180-199 42 42 42 66 66 66 T T F <---
58112024-02-08T06:41:35.175Z 10 200-219 52 52 50 82 82 76 F F F <---
58122024-02-08T06:41:35.179Z 11 220-239 49 49 49 75 75 75 F F F
58132024-02-08T06:41:35.179Z 12 240-259 52 52 48 79 79 73 F F F <---
58142024-02-08T06:41:35.179Z 13 260-279 52 52 46 81 81 70 F F F <---
58152024-02-08T06:41:35.179Z 14 280-299 50 50 50 76 76 76 F F F
58162024-02-08T06:41:35.179Z 15 300-319 52 52 50 81 81 77 F F F <---
58172024-02-08T06:41:35.179Z 16 320-339 50 50 50 77 77 77 F F F
58182024-02-08T06:41:35.179Z 17 340-359 52 52 44 82 82 68 F F F <---
58192024-02-08T06:41:35.179Z 18 360-379 47 47 47 72 72 72 F F F
58202024-02-08T06:41:35.179Z 19 380-399 52 52 39 80 80 59 F F F <---
58212024-02-08T06:41:35.179Z 20 400-419 46 46 46 70 70 70 F F F
58222024-02-08T06:41:35.180Z 21 420-439 46 46 46 70 70 70 T T F <---
58232024-02-08T06:41:35.180Z 22 440-459 51 51 51 78 78 78 F F F
58242024-02-08T06:41:35.180Z 23 460-479 52 52 49 79 79 75 F F F <---
58252024-02-08T06:41:35.180Z 24 480-499 52 52 41 79 79 64 F F F <---
58262024-02-08T06:41:35.180Z 25 500-519 52 52 42 82 82 65 F F F <---
58272024-02-08T06:41:35.180Z 26 520-539 49 49 49 75 75 75 F F F
58282024-02-08T06:41:35.180Z 27 540-559 52 52 50 79 79 76 F F T <---
58292024-02-08T06:41:35.180Z 28 560-579 42 42 42 65 65 65 T T F <---
58302024-02-08T06:41:35.180Z 29 580-599 50 50 50 76 76 76 F F F
58312024-02-08T06:41:35.180ZMax gen: 52, Max flush: 82
58322024-02-08T06:41:35.180ZError: Difference in extent metadata found!
58332024-02-08T06:41:35.180ZA Difference in extent metadata is expected here
58342024-02-08T06:41:35.180ZOn loop 26
58352024-02-08T06:41:35.180Z
58362024-02-08T06:41:35.184ZVerifying data now
58372024-02-08T06:41:40.839ZNo change after verify
58382024-02-08T06:41:40.842ZLoop: 26 Downstairs dump after verify (and repair):
58392024-02-08T06:41:40.854ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
58402024-02-08T06:41:40.855Z 0 000-019 40 40 40 62 62 62 F F F
58412024-02-08T06:41:40.855Z 1 020-039 44 44 44 68 68 68 F F F
58422024-02-08T06:41:40.855Z 2 040-059 49 49 49 75 75 75 F F F
58432024-02-08T06:41:40.855Z 3 060-079 52 52 52 82 82 82 F F F
58442024-02-08T06:41:40.855Z 4 080-099 46 46 46 71 71 71 F F F
58452024-02-08T06:41:40.855Z 5 100-119 51 51 51 78 78 78 F F F
58462024-02-08T06:41:40.855Z 6 120-139 48 48 48 73 73 73 F F F
58472024-02-08T06:41:40.855Z 7 140-159 51 51 51 78 78 78 F F F
58482024-02-08T06:41:40.855Z 8 160-179 51 51 51 78 78 78 F F F
58492024-02-08T06:41:40.855Z 9 180-199 53 53 53 83 83 83 F F F
58502024-02-08T06:41:40.855Z 10 200-219 52 52 52 82 82 82 F F F
58512024-02-08T06:41:40.858Z 11 220-239 49 49 49 75 75 75 F F F
58522024-02-08T06:41:40.859Z 12 240-259 52 52 52 79 79 79 F F F
58532024-02-08T06:41:40.859Z 13 260-279 52 52 52 81 81 81 F F F
58542024-02-08T06:41:40.859Z 14 280-299 50 50 50 76 76 76 F F F
58552024-02-08T06:41:40.859Z 15 300-319 52 52 52 81 81 81 F F F
58562024-02-08T06:41:40.859Z 16 320-339 50 50 50 77 77 77 F F F
58572024-02-08T06:41:40.859Z 17 340-359 52 52 52 82 82 82 F F F
58582024-02-08T06:41:40.859Z 18 360-379 47 47 47 72 72 72 F F F
58592024-02-08T06:41:40.859Z 19 380-399 52 52 52 80 80 80 F F F
58602024-02-08T06:41:40.859Z 20 400-419 46 46 46 70 70 70 F F F
58612024-02-08T06:41:40.859Z 21 420-439 53 53 53 83 83 83 F F F
58622024-02-08T06:41:40.859Z 22 440-459 51 51 51 78 78 78 F F F
58632024-02-08T06:41:40.859Z 23 460-479 52 52 52 79 79 79 F F F
58642024-02-08T06:41:40.859Z 24 480-499 52 52 52 79 79 79 F F F
58652024-02-08T06:41:40.859Z 25 500-519 52 52 52 82 82 82 F F F
58662024-02-08T06:41:40.860Z 26 520-539 49 49 49 75 75 75 F F F
58672024-02-08T06:41:40.860Z 27 540-559 52 52 52 79 79 79 F F F
58682024-02-08T06:41:40.860Z 28 560-579 53 53 53 83 83 83 F F F
58692024-02-08T06:41:40.860Z 29 580-599 50 50 50 76 76 76 F F F
58702024-02-08T06:41:40.860ZMax gen: 53, Max flush: 83
58712024-02-08T06:41:40.860Z
58722024-02-08T06:41:40.860ZBegin loop 27
58732024-02-08T06:41:40.860ZDownstairs to restart: 0
58742024-02-08T06:41:40.875Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.11809359Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354}
58752024-02-08T06:41:40.875Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118351301Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354}
58762024-02-08T06:41:40.875Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118377309Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354}
58772024-02-08T06:41:40.882Z{"msg":"Crucible b6ce46f7-6a1d-419d-89da-f7cbc2f77f64 has session id: 30cab1e8-4377-407a-851d-a00ebf46b07d","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118407567Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58782024-02-08T06:41:40.882ZCrucible runtime is spawned
58792024-02-08T06:41:40.882Z{"msg":"Upstairs opts: Upstairs UUID: b6ce46f7-6a1d-419d-89da-f7cbc2f77f64, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118465393Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58802024-02-08T06:41:40.882Z{"msg":"Crucible stats registered with UUID: b6ce46f7-6a1d-419d-89da-f7cbc2f77f64","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118515609Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58812024-02-08T06:41:40.882Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118558546Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354}
58822024-02-08T06:41:40.883Z{"msg":"b6ce46f7-6a1d-419d-89da-f7cbc2f77f64 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118687316Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58832024-02-08T06:41:40.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118724733Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58842024-02-08T06:41:40.883Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118800028Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58852024-02-08T06:41:40.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118862193Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58862024-02-08T06:41:40.883Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.11889564Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58872024-02-08T06:41:40.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118940407Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58882024-02-08T06:41:40.883Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.118973954Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58892024-02-08T06:41:40.883Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.119007032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"io task","client":"0","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58902024-02-08T06:41:40.883Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.119042549Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"io task","client":"2","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58912024-02-08T06:41:40.883Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.119092295Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"io task","client":"1","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58922024-02-08T06:41:40.883Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.119129003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"io task","client":"2","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58932024-02-08T06:41:40.883Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.119171489Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"io task","client":"0","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58942024-02-08T06:41:40.883Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.119217346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"io task","client":"1","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58952024-02-08T06:41:40.883Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.119717179Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58962024-02-08T06:41:40.884Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.119756396Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58972024-02-08T06:41:40.884Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.119781574Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58982024-02-08T06:41:40.884Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.119804492Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
58992024-02-08T06:41:40.884Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.120412387Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59002024-02-08T06:41:40.884Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.120454053Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59012024-02-08T06:41:40.884Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.120879001Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59022024-02-08T06:41:40.884Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.120938957Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59032024-02-08T06:41:40.884Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.121113984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59042024-02-08T06:41:40.884Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.121151481Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59052024-02-08T06:41:40.884Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.12116839Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59062024-02-08T06:41:40.884Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.121193788Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59072024-02-08T06:41:40.884Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.121210956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59082024-02-08T06:41:40.884Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.121237374Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59092024-02-08T06:41:40.884Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.121512964Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59102024-02-08T06:41:40.888Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122019156Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59112024-02-08T06:41:40.888Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122067882Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59122024-02-08T06:41:40.888Z{"msg":"[0]R flush_numbers[0..12]: [62, 68, 75, 82, 71, 78, 73, 78, 78, 83, 82, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122105239Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59132024-02-08T06:41:40.889Z{"msg":"[0]R generation[0..12]: [40, 44, 49, 52, 46, 51, 48, 51, 51, 53, 52, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122141586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59142024-02-08T06:41:40.889Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122177634Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59152024-02-08T06:41:40.889Z{"msg":"[1]R flush_numbers[0..12]: [62, 68, 75, 82, 71, 78, 73, 78, 78, 83, 82, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122212471Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59162024-02-08T06:41:40.889Z{"msg":"[1]R generation[0..12]: [40, 44, 49, 52, 46, 51, 48, 51, 51, 53, 52, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122248058Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,Wait for a query_work_queue command to finish before sending IO
59172024-02-08T06:41:40.889Z"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59182024-02-08T06:41:40.889Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122281496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59192024-02-08T06:41:40.889Z{"msg":"[2]R flush_numbers[0..12]: [62, 68, 75, 82, 71, 78, 73, 78, 78, 83, 82, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122321023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59202024-02-08T06:41:40.889Z{"msg":"[2]R generation[0..12]: [40, 44, 49, 52, 46, 51, 48, 51, 51, 53, 52, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.12235819Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59212024-02-08T06:41:40.889Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122396427Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59222024-02-08T06:41:40.889Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122432725Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59232024-02-08T06:41:40.889Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122469042Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59242024-02-08T06:41:40.889Z{"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122506219Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59252024-02-08T06:41:40.889Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122542816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59262024-02-08T06:41:40.889ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
59272024-02-08T06:41:40.889Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122604362Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59282024-02-08T06:41:40.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122664957Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59292024-02-08T06:41:40.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122687885Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59302024-02-08T06:41:40.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122702024Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59312024-02-08T06:41:40.890Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122722523Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59322024-02-08T06:41:40.890Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122735642Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59332024-02-08T06:41:40.890Z{"msg":"b6ce46f7-6a1d-419d-89da-f7cbc2f77f64 is now active with session: 30cab1e8-4377-407a-851d-a00ebf46b07d","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.12275534Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59342024-02-08T06:41:40.890Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122767859Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354,"session_id":"30cab1e8-4377-407a-851d-a00ebf46b07d"}
59352024-02-08T06:41:40.890Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:40.122788758Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354}
59362024-02-08T06:41:40.890ZLoading write count information from file "/tmp/test_repair_verify.data"
59372024-02-08T06:41:40.890ZRun Repair workload
59382024-02-08T06:41:40.890Z01/30 Write block 76 len 16384 data: 5 3 4 4
59392024-02-08T06:41:40.890Z02/30 Read block 562 len 16384
59402024-02-08T06:41:40.890Z03/30 Write block 533 len 24576 data: 8 8 7 8 7 6
59412024-02-08T06:41:40.890Z04/30 Read block 222 len 12288
59422024-02-08T06:41:40.890Z05/30 Read block 509 len 36864
59432024-02-08T06:41:40.891Z06/30 Read block 115 len 20480
59442024-02-08T06:41:40.891Z07/30 Write block 449 len 28672 data: 7 6 5 5 5 4 6
59452024-02-08T06:41:40.891Z08/30 Read block 429 len 12288
59462024-02-08T06:41:40.894Z09/30 Read block 424 len 36864
59472024-02-08T06:41:40.894Z10/30 Read block 183 len 28672
59482024-02-08T06:41:40.894Z11/30 Write block 405 len 12288 data: 6 5 5
59492024-02-08T06:41:40.894Z12/30 Read block 361 len 28672
59502024-02-08T06:41:40.894Z13/30 Read block 212 len 24576
59512024-02-08T06:41:40.894Z14/30 Flush
59522024-02-08T06:41:40.897Z15/30 Write block 568 len 4096 data: 5
59532024-02-08T06:41:40.897Z16/30 Write block 336 len 28672 data: 8 8 8 7 6 5 4
59542024-02-08T06:41:40.897Z17/30 Read block 214 len 32768
59552024-02-08T06:41:40.900Z18/30 Read block 378 len 12288
59562024-02-08T06:41:40.900Z19/30 Write block 292 len 8192 data: 6 5
59572024-02-08T06:41:40.900Z20/30 Read block 50 len 24576
59582024-02-08T06:41:40.900Z21/30 Flush
59592024-02-08T06:41:40.903Z22/30 Write block 380 len 4096 data: 4
59602024-02-08T06:41:40.903Z23/30 Read block 150 len 36864
59612024-02-08T06:41:40.906Z24/30 Write block 23 len 24576 data: 6 7 8 7 7 6
59622024-02-08T06:41:40.906Z25/30 Flush
59632024-02-08T06:41:40.912Z26/30 Write block 582 len 32768 data: 3 5 5 3 3 4 5 7
59642024-02-08T06:41:40.915Z27/30 Read block 187 len 40960
59652024-02-08T06:41:40.915Z28/30 Write block 399 len 20480 data: 4 5 6 6 6
59662024-02-08T06:41:40.915Z29/30 Read block 144 len 36864
59672024-02-08T06:41:40.918Z30/30 Write block 210 len 20480 data: 3 5 7 7 7
59682024-02-08T06:41:40.918Z----------------------------------------------------------------
59692024-02-08T06:41:40.918Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59702024-02-08T06:41:40.918ZStates: Active Active Active
59712024-02-08T06:41:40.918ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59722024-02-08T06:41:40.918Z 1 Acked 1000 Write 4 Done Done Done false
59732024-02-08T06:41:40.918Z 2 Acked 1001 Read 4 Done Done Done false
59742024-02-08T06:41:40.918Z 3 Acked 1002 Write 6 Sent Done Done false
59752024-02-08T06:41:40.918Z 4 Acked 1003 Read 3 Sent Done Done false
59762024-02-08T06:41:40.919Z 5 Acked 1004 Read 9 Sent Done Done false
59772024-02-08T06:41:40.919Z 6 Acked 1005 Read 5 Sent Done Done false
59782024-02-08T06:41:40.919Z 7 Acked 1006 Write 7 Sent Done Done false
59792024-02-08T06:41:40.919Z 8 Acked 1007 Read 3 Sent Done Done false
59802024-02-08T06:41:40.919Z 9 Acked 1008 Read 9 Sent Done Done false
59812024-02-08T06:41:40.919Z 10 Acked 1009 Read 7 Sent Done Done false
59822024-02-08T06:41:40.919Z 11 Acked 1010 Write 3 Sent Done Done false
59832024-02-08T06:41:40.919Z 12 Acked 1011 Read 7 Sent Done Done false
59842024-02-08T06:41:40.919Z 13 Acked 1012 Read 6 Sent Done Done false
59852024-02-08T06:41:40.919Z 14 Acked 1013 Flush 0 Sent Done Done false
59862024-02-08T06:41:40.919Z 15 Acked 1014 Write 1 Sent Done Done false
59872024-02-08T06:41:40.919Z 16 Acked 1015 Write 7 Sent Done Done false
59882024-02-08T06:41:40.919Z 17 Acked 1016 Read 8 Sent Done Done false
59892024-02-08T06:41:40.919Z 18 Acked 1017 Read 3 Sent Done Done false
59902024-02-08T06:41:40.919Z 19 Acked 1018 Write 2 Sent Done Done false
59912024-02-08T06:41:40.920Z 20 Acked 1019 Read 6 Sent Done Done false
59922024-02-08T06:41:40.920Z 21 Acked 1020 Flush 0 Sent Done Done false
59932024-02-08T06:41:40.920Z 22 Acked 1021 Write 1 Sent Done Done false
59942024-02-08T06:41:40.920Z 23 Acked 1022 Read 9 Sent Done Done false
59952024-02-08T06:41:40.920Z 24 Acked 1023 Write 6 Sent Done Done false
59962024-02-08T06:41:40.920Z 25 Acked 1024 Flush 0 Sent Done Done false
59972024-02-08T06:41:40.920Z 26 Acked 1025 Write 8 Sent Done Done false
59982024-02-08T06:41:40.920Z 27 Acked 1026 Read 10 Sent Done Done false
59992024-02-08T06:41:40.920Z 28 Acked 1027 Write 5 Sent Done Done false
60002024-02-08T06:41:40.920Z 29 Acked 1028 Read 9 Sent Done Done false
60012024-02-08T06:41:40.920Z 30 Acked 1029 Write 5 Sent Sent Sent false
60022024-02-08T06:41:40.920Z STATES DS:0 DS:1 DS:2 TOTAL
60032024-02-08T06:41:40.920Z New 0 0 0 0
60042024-02-08T06:41:40.920Z Sent 28 1 1 30
60052024-02-08T06:41:40.920Z Done 2 29 29 60
60062024-02-08T06:41:40.921Z Skipped 0 0 0 0
60072024-02-08T06:41:40.921Z Error 0 0 0 0
60082024-02-08T06:41:40.921ZLast Flush: 0 1024 1024
60092024-02-08T06:41:40.921ZDownstairs last five completed:
60102024-02-08T06:41:40.921ZUpstairs last five completed: 30 29 28 27 26
60112024-02-08T06:41:40.921Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:40.159574326Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1354}
60122024-02-08T06:41:40.921ZWrote out file "/tmp/test_repair_verify.data"
60132024-02-08T06:41:40.924Z
60142024-02-08T06:41:40.924ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
60152024-02-08T06:41:40.940ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60162024-02-08T06:41:40.940Z 0 000-019 40 40 40 62 62 62 F F F
60172024-02-08T06:41:40.940Z 1 020-039 44 54 54 68 86 86 F F F <---
60182024-02-08T06:41:40.940Z 2 040-059 49 49 49 75 75 75 F F F
60192024-02-08T06:41:40.940Z 3 060-079 52 54 54 82 84 84 T F F <---
60202024-02-08T06:41:40.940Z 4 080-099 46 46 46 71 71 71 F F F
60212024-02-08T06:41:40.940Z 5 100-119 51 51 51 78 78 78 F F F
60222024-02-08T06:41:40.940Z 6 120-139 48 48 48 73 73 73 F F F
60232024-02-08T06:41:40.940Z 7 140-159 51 51 51 78 78 78 F F F
60242024-02-08T06:41:40.944Z 8 160-179 51 51 51 78 78 78 F F F
60252024-02-08T06:41:40.944Z 9 180-199 53 53 53 83 83 83 F F F
60262024-02-08T06:41:40.944Z 10 200-219 52 52 52 82 82 82 F T T <---
60272024-02-08T06:41:40.944Z 11 220-239 49 49 49 75 75 75 F F F
60282024-02-08T06:41:40.944Z 12 240-259 52 52 52 79 79 79 F F F
60292024-02-08T06:41:40.944Z 13 260-279 52 52 52 81 81 81 F F F
60302024-02-08T06:41:40.944Z 14 280-299 50 54 54 76 85 85 F F F <---
60312024-02-08T06:41:40.944Z 15 300-319 52 52 52 81 81 81 F F F
60322024-02-08T06:41:40.944Z 16 320-339 50 54 54 77 85 85 F F F <---
60332024-02-08T06:41:40.944Z 17 340-359 52 54 54 82 85 85 F F F <---
60342024-02-08T06:41:40.944Z 18 360-379 47 47 47 72 72 72 F F F
60352024-02-08T06:41:40.944Z 19 380-399 52 54 54 80 86 86 F T T <---
60362024-02-08T06:41:40.944Z 20 400-419 46 54 54 70 84 84 F T T <---
60372024-02-08T06:41:40.944Z 21 420-439 53 53 53 83 83 83 F F F
60382024-02-08T06:41:40.944Z 22 440-459 51 54 54 78 84 84 F F F <---
60392024-02-08T06:41:40.945Z 23 460-479 52 52 52 79 79 79 F F F
60402024-02-08T06:41:40.945Z 24 480-499 52 52 52 79 79 79 F F F
60412024-02-08T06:41:40.945Z 25 500-519 52 52 52 82 82 82 F F F
60422024-02-08T06:41:40.945Z 26 520-539 49 54 54 75 84 84 F F F <---
60432024-02-08T06:41:40.945Z 27 540-559 52 52 52 79 79 79 F F F
60442024-02-08T06:41:40.945Z 28 560-579 53 54 54 83 85 85 F F F <---
60452024-02-08T06:41:40.945Z 29 580-599 50 50 50 76 76 76 F T T <---
60462024-02-08T06:41:40.945ZMax gen: 54, Max flush: 86
60472024-02-08T06:41:40.945ZError: Difference in extent metadata found!
60482024-02-08T06:41:40.945ZA Difference in extent metadata is expected here
60492024-02-08T06:41:40.945ZOn loop 27
60502024-02-08T06:41:40.945Z
60512024-02-08T06:41:40.948ZVerifying data now
60522024-02-08T06:41:45.533ZNo change after verify
60532024-02-08T06:41:45.536ZLoop: 27 Downstairs dump after verify (and repair):
60542024-02-08T06:41:45.548ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60552024-02-08T06:41:45.548Z 0 000-019 40 40 40 62 62 62 F F F
60562024-02-08T06:41:45.548Z 1 020-039 54 54 54 86 86 86 F F F
60572024-02-08T06:41:45.548Z 2 040-059 49 49 49 75 75 75 F F F
60582024-02-08T06:41:45.548Z 3 060-079 54 54 54 84 84 84 F F F
60592024-02-08T06:41:45.548Z 4 080-099 46 46 46 71 71 71 F F F
60602024-02-08T06:41:45.548Z 5 100-119 51 51 51 78 78 78 F F F
60612024-02-08T06:41:45.548Z 6 120-139 48 48 48 73 73 73 F F F
60622024-02-08T06:41:45.548Z 7 140-159 51 51 51 78 78 78 F F F
60632024-02-08T06:41:45.548Z 8 160-179 51 51 51 78 78 78 F F F
60642024-02-08T06:41:45.548Z 9 180-199 53 53 53 83 83 83 F F F
60652024-02-08T06:41:45.552Z 10 200-219 55 55 55 87 87 87 F F F
60662024-02-08T06:41:45.552Z 11 220-239 49 49 49 75 75 75 F F F
60672024-02-08T06:41:45.552Z 12 240-259 52 52 52 79 79 79 F F F
60682024-02-08T06:41:45.552Z 13 260-279 52 52 52 81 81 81 F F F
60692024-02-08T06:41:45.552Z 14 280-299 54 54 54 85 85 85 F F F
60702024-02-08T06:41:45.552Z 15 300-319 52 52 52 81 81 81 F F F
60712024-02-08T06:41:45.552Z 16 320-339 54 54 54 85 85 85 F F F
60722024-02-08T06:41:45.552Z 17 340-359 54 54 54 85 85 85 F F F
60732024-02-08T06:41:45.552Z 18 360-379 47 47 47 72 72 72 F F F
60742024-02-08T06:41:45.552Z 19 380-399 55 55 55 87 87 87 F F F
60752024-02-08T06:41:45.552Z 20 400-419 55 55 55 87 87 87 F F F
60762024-02-08T06:41:45.553Z 21 420-439 53 53 53 83 83 83 F F F
60772024-02-08T06:41:45.553Z 22 440-459 54 54 54 84 84 84 F F F
60782024-02-08T06:41:45.553Z 23 460-479 52 52 52 79 79 79 F F F
60792024-02-08T06:41:45.553Z 24 480-499 52 52 52 79 79 79 F F F
60802024-02-08T06:41:45.553Z 25 500-519 52 52 52 82 82 82 F F F
60812024-02-08T06:41:45.553Z 26 520-539 54 54 54 84 84 84 F F F
60822024-02-08T06:41:45.553Z 27 540-559 52 52 52 79 79 79 F F F
60832024-02-08T06:41:45.553Z 28 560-579 54 54 54 85 85 85 F F F
60842024-02-08T06:41:45.553Z 29 580-599 55 55 55 87 87 87 F F F
60852024-02-08T06:41:45.553ZMax gen: 55, Max flush: 87
60862024-02-08T06:41:45.553Z
60872024-02-08T06:41:45.553ZBegin loop 28
60882024-02-08T06:41:45.553ZDownstairs to restart: 1
60892024-02-08T06:41:45.569Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.810913614Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362}
60902024-02-08T06:41:45.569Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811206241Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362}
60912024-02-08T06:41:45.576Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
60922024-02-08T06:41:45.576Z,"time":"2024-02-08T06:41:44.811248189Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362}
60932024-02-08T06:41:45.576Z{"msg":"Crucible be1996c5-cc46-44d8-9ab9-0f9b0a479d43 has session id: 3924d347-d027-4b72-9b9f-18fd88d1dd50","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811287426Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
60942024-02-08T06:41:45.576Z{"msg":"Upstairs opts: Upstairs UUID: be1996c5-cc46-44d8-9ab9-0f9b0a479d43, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811330332Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
60952024-02-08T06:41:45.576Z{"msg":"Crucible stats registered with UUID: be1996c5-cc46-44d8-9ab9-0f9b0a479d43","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811370939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
60962024-02-08T06:41:45.576Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811395277Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362}
60972024-02-08T06:41:45.576Z{"msg":"be1996c5-cc46-44d8-9ab9-0f9b0a479d43 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811511059Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
60982024-02-08T06:41:45.576Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811559265Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
60992024-02-08T06:41:45.576Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811604512Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61002024-02-08T06:41:45.576Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811652058Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61012024-02-08T06:41:45.576Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811708244Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61022024-02-08T06:41:45.576Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811732872Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61032024-02-08T06:41:45.576Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.81175392Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61042024-02-08T06:41:45.577Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811812456Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"io task","client":"0","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61052024-02-08T06:41:45.577Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811851253Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"io task","client":"1","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61062024-02-08T06:41:45.577Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.81189974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"io task","client":"2","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61072024-02-08T06:41:45.577Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811941656Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"io task","client":"0","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61082024-02-08T06:41:45.577Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.811981333Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"io task","client":"2","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61092024-02-08T06:41:45.577Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.812007791Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"io task","client":"1","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61102024-02-08T06:41:45.577Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.812620266Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61112024-02-08T06:41:45.577Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.812701219Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61122024-02-08T06:41:45.577Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.812726818Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61132024-02-08T06:41:45.577Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.812752745Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61142024-02-08T06:41:45.577Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.813169164Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61152024-02-08T06:41:45.577Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.813330642Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61162024-02-08T06:41:45.577Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.813655618Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61172024-02-08T06:41:45.577Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.813692485Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61182024-02-08T06:41:45.577Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.813718923Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61192024-02-08T06:41:45.578Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.81375458Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61202024-02-08T06:41:45.578Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.813776848Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61212024-02-08T06:41:45.578Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.813808386Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61222024-02-08T06:41:45.578Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.813830944Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61232024-02-08T06:41:45.578Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.813848963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61242024-02-08T06:41:45.578Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.814329727Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61252024-02-08T06:41:45.583Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.814800252Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61262024-02-08T06:41:45.583Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.81482982Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61272024-02-08T06:41:45.583Z{"msg":"[0]R flush_numbers[0..12]: [62, 86, 75, 84, 71, 78, 73, 78, 78, 83, 87, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.814844859Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61282024-02-08T06:41:45.583Z{"msg":"[0]R generation[0..12]: [40, 54, 49, 54, 46, 51, 48, 51, 51, 53, 55, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.814859117Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61292024-02-08T06:41:45.583Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.814874296Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61302024-02-08T06:41:45.583Z{"msg":"[1]R flush_numbers[0..12]: [62, 86, 75, 84, 71, 78, 73, 78, 78, 83, 87, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.814888505Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61312024-02-08T06:41:45.583Z{"msg":"[1]R generation[0..12]: [40, 54, 49, 54, 46, 51, 48, 51, 51, 53, 55, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.814914163Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61322024-02-08T06:41:45.583Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.814931372Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61332024-02-08T06:41:45.583Z{"msg":"[2]R flush_numbers[0..12]: [62, 86, 75, 84, 71, 78, 73, 78, 78, 83, 87, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.8149533Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61342024-02-08T06:41:45.583Z{"msg":"[2]R generation[0..12]: [40, 54, 49, 54, 46, 51, 48, 51, 51, 53, 55, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.814967319Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61352024-02-08T06:41:45.583Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.814988088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61362024-02-08T06:41:45.583Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
61372024-02-08T06:41:45.583Z,"time":"2024-02-08T06:41:44.815002256Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61382024-02-08T06:41:45.583Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.815025585Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61392024-02-08T06:41:45.584Z{"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.815038974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61402024-02-08T06:41:45.584Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.815058752Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61412024-02-08T06:41:45.584Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.815072141Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61422024-02-08T06:41:45.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.81509092Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61432024-02-08T06:41:45.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.815104509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61442024-02-08T06:41:45.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.815124078Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61452024-02-08T06:41:45.584Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.815140826Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61462024-02-08T06:41:45.584Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.815173424Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61472024-02-08T06:41:45.584Z{"msg":"be1996c5-cc46-44d8-9ab9-0f9b0a479d43 is now active with session: 3924d347-d027-4b72-9b9f-18fd88d1dd50","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.815206541Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61482024-02-08T06:41:45.584Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.81522578Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362,"session_id":"3924d347-d027-4b72-9b9f-18fd88d1dd50"}
61492024-02-08T06:41:45.584Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:44.815251518Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362}
61502024-02-08T06:41:45.584ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
61512024-02-08T06:41:45.584ZLoading write count information from file "/tmp/test_repair_verify.data"
61522024-02-08T06:41:45.584ZRun Repair workload
61532024-02-08T06:41:45.584Z01/30 Flush
61542024-02-08T06:41:45.585Z02/30 Write block 264 len 40960 data: 4 4 4 5 7 7 5 6 6 6
61552024-02-08T06:41:45.585Z03/30 Read block 382 len 40960
61562024-02-08T06:41:45.585Z04/30 Write block 484 len 24576 data: 6 6 7 7 7 6
61572024-02-08T06:41:45.585Z05/30 Read block 24 len 24576
61582024-02-08T06:41:45.585Z06/30 Read block 456 len 36864
61592024-02-08T06:41:45.585Z07/30 Read block 108 len 28672
61602024-02-08T06:41:45.585Z08/30 Read block 407 len 12288
61612024-02-08T06:41:45.588Z09/30 Write block 320 len 32768 data: 8 6 6 6 4 5 3 3
61622024-02-08T06:41:45.588Z10/30 Write block 163 len 40960 data: 6 7 6 6 6 5 6 6 6 7
61632024-02-08T06:41:45.589Z11/30 Flush
61642024-02-08T06:41:45.589Z12/30 Write block 485 len 28672 data: 7 8 8 8 7 5 5
61652024-02-08T06:41:45.589Z13/30 Write block 518 len 16384 data: 5 6 5 4
61662024-02-08T06:41:45.589Z14/30 Read block 8 len 32768
61672024-02-08T06:41:45.592Z15/30 Flush
61682024-02-08T06:41:45.592Z16/30 Write block 553 len 20480 data: 6 6 5 8 6
61692024-02-08T06:41:45.592Z17/30 Read block 251 len 28672
61702024-02-08T06:41:45.596Z18/30 Write block 99 len 12288 data: 4 5 5
61712024-02-08T06:41:45.596Z19/30 Read block 103 len 24576
61722024-02-08T06:41:45.596Z20/30 Write block 511 len 4096 data: 4
61732024-02-08T06:41:45.597Z21/30 Write block 23 len 16384 data: 7 8 9 8
61742024-02-08T06:41:45.597Z22/30 Write block 224 len 8192 data: 11 10
61752024-02-08T06:41:45.599Z23/30 Flush
61762024-02-08T06:41:45.599Z24/30 Write block 519 len 8192 data: 7 6
61772024-02-08T06:41:45.600Z25/30 Write block 436 len 32768 data: 8 8 6 4 6 5 5 6
61782024-02-08T06:41:45.600Z26/30 Read block 44 len 36864
61792024-02-08T06:41:45.603Z27/30 Write block 552 len 4096 data: 5
61802024-02-08T06:41:45.603Z28/30 Read block 339 len 20480
61812024-02-08T06:41:45.603Z29/30 Read block 328 len 24576
61822024-02-08T06:41:45.603Z30/30 Write block 193 len 12288 data: 7 8 6
61832024-02-08T06:41:45.603Z----------------------------------------------------------------
61842024-02-08T06:41:45.603Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61852024-02-08T06:41:45.603ZStates: Active Active Active
61862024-02-08T06:41:45.603ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61872024-02-08T06:41:45.603Z 1 Acked 1000 Flush 0 Done Sent Done false
61882024-02-08T06:41:45.603Z 2 Acked 1001 Write 10 Done Sent Done false
61892024-02-08T06:41:45.603Z 3 Acked 1002 Read 10 Done Sent Done false
61902024-02-08T06:41:45.603Z 4 Acked 1003 Write 6 Done Sent Done false
61912024-02-08T06:41:45.604Z 5 Acked 1004 Read 6 Done Sent Done false
61922024-02-08T06:41:45.604Z 6 Acked 1005 Read 9 Done Sent Done false
61932024-02-08T06:41:45.607Z 7 Acked 1006 Read 7 Done Sent Done false
61942024-02-08T06:41:45.607Z 8 Acked 1007 Read 3 Done Sent Done false
61952024-02-08T06:41:45.607Z 9 Acked 1008 Write 8 Done Sent Done false
61962024-02-08T06:41:45.607Z 10 Acked 1009 Write 10 Done Sent Done false
61972024-02-08T06:41:45.607Z 11 Acked 1010 Flush 0 Done Sent Done false
61982024-02-08T06:41:45.607Z 12 Acked 1011 Write 7 Done Sent Done false
61992024-02-08T06:41:45.608Z 13 Acked 1012 Write 4 Done Sent Done false
62002024-02-08T06:41:45.608Z 14 Acked 1013 Read 8 Done Sent Done false
62012024-02-08T06:41:45.608Z 15 Acked 1014 Flush 0 Done Sent Done false
62022024-02-08T06:41:45.608Z 16 Acked 1015 Write 5 Done Sent Done false
62032024-02-08T06:41:45.608Z 17 Acked 1016 Read 7 Done Sent Done false
62042024-02-08T06:41:45.608Z 18 Acked 1017 Write 3 Done Sent Done false
62052024-02-08T06:41:45.608Z 19 Acked 1018 Read 6 Done Sent Done false
62062024-02-08T06:41:45.608Z 20 Acked 1019 Write 1 Done Sent Done false
62072024-02-08T06:41:45.608Z 21 Acked 1020 Write 4 Done Sent Done false
62082024-02-08T06:41:45.608Z 22 Acked 1021 Write 2 Done Sent Done false
62092024-02-08T06:41:45.608Z 23 Acked 1022 Flush 0 Done Sent Done false
62102024-02-08T06:41:45.608Z 24 Acked 1023 Write 2 Done Sent Done false
62112024-02-08T06:41:45.608Z 25 Acked 1024 Write 8 Done Sent Done false
62122024-02-08T06:41:45.608Z 26 Acked 1025 Read 9 Done Sent Done false
62132024-02-08T06:41:45.608Z 27 Acked 1026 Write 1 Done Sent Done false
62142024-02-08T06:41:45.609Z 28 Acked 1027 Read 5 Done Sent Done false
62152024-02-08T06:41:45.609Z 29 Acked 1028 Read 6 Done Sent Done false
62162024-02-08T06:41:45.609Z 30 Acked 1029 Write 3 Sent Sent Sent false
62172024-02-08T06:41:45.609Z STATES DS:0 DS:1 DS:2 TOTAL
62182024-02-08T06:41:45.609Z New 0 0 0 0
62192024-02-08T06:41:45.609Z Sent 1 30 1 32
62202024-02-08T06:41:45.609Z Done 29 0 29 58
62212024-02-08T06:41:45.609Z Skipped 0 0 0 0
62222024-02-08T06:41:45.609Z Error 0 0 0 0
62232024-02-08T06:41:45.609ZLast Flush: 1022 0 1022
62242024-02-08T06:41:45.609ZDownstairs last five completed:
62252024-02-08T06:41:45.609ZUpstairs last five completed: 30 29 28 27 26
62262024-02-08T06:41:45.609Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:44.845885688Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1362}
62272024-02-08T06:41:45.609ZWrote out file "/tmp/test_repair_verify.data"
62282024-02-08T06:41:45.609Z
62292024-02-08T06:41:45.615ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
62302024-02-08T06:41:45.627ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62312024-02-08T06:41:45.627Z 0 000-019 40 40 40 62 62 62 F F F
62322024-02-08T06:41:45.627Z 1 020-039 56 54 56 91 86 91 F F F <---
62332024-02-08T06:41:45.627Z 2 040-059 49 49 49 75 75 75 F F F
62342024-02-08T06:41:45.632Z 3 060-079 54 54 54 84 84 84 F F F
62352024-02-08T06:41:45.632Z 4 080-099 56 46 56 91 71 91 F F F <---
62362024-02-08T06:41:45.632Z 5 100-119 56 51 56 91 78 91 F F F <---
62372024-02-08T06:41:45.632Z 6 120-139 48 48 48 73 73 73 F F F
62382024-02-08T06:41:45.632Z 7 140-159 51 51 51 78 78 78 F F F
62392024-02-08T06:41:45.632Z 8 160-179 56 51 56 89 78 89 F F F <---
62402024-02-08T06:41:45.632Z 9 180-199 53 53 53 83 83 83 T F T <---
62412024-02-08T06:41:45.632Z 10 200-219 55 55 55 87 87 87 F F F
62422024-02-08T06:41:45.633Z 11 220-239 56 49 56 91 75 91 F F F <---
62432024-02-08T06:41:45.633Z 12 240-259 52 52 52 79 79 79 F F F
62442024-02-08T06:41:45.633Z 13 260-279 56 52 56 89 81 89 F F F <---
62452024-02-08T06:41:45.633Z 14 280-299 54 54 54 85 85 85 F F F
62462024-02-08T06:41:45.633Z 15 300-319 52 52 52 81 81 81 F F F
62472024-02-08T06:41:45.633Z 16 320-339 56 54 56 89 85 89 F F F <---
62482024-02-08T06:41:45.633Z 17 340-359 54 54 54 85 85 85 F F F
62492024-02-08T06:41:45.633Z 18 360-379 47 47 47 72 72 72 F F F
62502024-02-08T06:41:45.633Z 19 380-399 55 55 55 87 87 87 F F F
62512024-02-08T06:41:45.633Z 20 400-419 55 55 55 87 87 87 F F F
62522024-02-08T06:41:45.633Z 21 420-439 53 53 53 83 83 83 T F T <---
62532024-02-08T06:41:45.633Z 22 440-459 54 54 54 84 84 84 T F T <---
62542024-02-08T06:41:45.633Z 23 460-479 52 52 52 79 79 79 F F F
62552024-02-08T06:41:45.633Z 24 480-499 56 52 56 90 79 90 F F F <---
62562024-02-08T06:41:45.633Z 25 500-519 56 52 56 91 82 91 T F T <---
62572024-02-08T06:41:45.634Z 26 520-539 56 54 56 90 84 90 T F T <---
62582024-02-08T06:41:45.634Z 27 540-559 56 52 56 91 79 91 T F T <---
62592024-02-08T06:41:45.634Z 28 560-579 54 54 54 85 85 85 F F F
62602024-02-08T06:41:45.634Z 29 580-599 55 55 55 87 87 87 F F F
62612024-02-08T06:41:45.634ZMax gen: 56, Max flush: 91
62622024-02-08T06:41:45.634ZError: Difference in extent metadata found!
62632024-02-08T06:41:45.634ZA Difference in extent metadata is expected here
62642024-02-08T06:41:45.634ZOn loop 28
62652024-02-08T06:41:45.634Z
62662024-02-08T06:41:45.637ZVerifying data now
62672024-02-08T06:41:50.322ZNo change after verify
62682024-02-08T06:41:50.323ZLoop: 28 Downstairs dump after verify (and repair):
62692024-02-08T06:41:50.337ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62702024-02-08T06:41:50.338Z 0 000-019 40 40 40 62 62 62 F F F
62712024-02-08T06:41:50.338Z 1 020-039 56 56 56 91 91 91 F F F
62722024-02-08T06:41:50.338Z 2 040-059 49 49 49 75 75 75 F F F
62732024-02-08T06:41:50.338Z 3 060-079 54 54 54 84 84 84 F F F
62742024-02-08T06:41:50.338Z 4 080-099 56 56 56 91 91 91 F F F
62752024-02-08T06:41:50.338Z 5 100-119 56 56 56 91 91 91 F F F
62762024-02-08T06:41:50.338Z 6 120-139 48 48 48 73 73 73 F F F
62772024-02-08T06:41:50.341Z 7 140-159 51 51 51 78 78 78 F F F
62782024-02-08T06:41:50.341Z 8 160-179 56 56 56 89 89 89 F F F
62792024-02-08T06:41:50.341Z 9 180-199 57 57 57 92 92 92 F F F
62802024-02-08T06:41:50.342Z 10 200-219 55 55 55 87 87 87 F F F
62812024-02-08T06:41:50.342Z 11 220-239 56 56 56 91 91 91 F F F
62822024-02-08T06:41:50.342Z 12 240-259 52 52 52 79 79 79 F F F
62832024-02-08T06:41:50.342Z 13 260-279 56 56 56 89 89 89 F F F
62842024-02-08T06:41:50.342Z 14 280-299 54 54 54 85 85 85 F F F
62852024-02-08T06:41:50.342Z 15 300-319 52 52 52 81 81 81 F F F
62862024-02-08T06:41:50.342Z 16 320-339 56 56 56 89 89 89 F F F
62872024-02-08T06:41:50.342Z 17 340-359 54 54 54 85 85 85 F F F
62882024-02-08T06:41:50.342Z 18 360-379 47 47 47 72 72 72 F F F
62892024-02-08T06:41:50.342Z 19 380-399 55 55 55 87 87 87 F F F
62902024-02-08T06:41:50.342Z 20 400-419 55 55 55 87 87 87 F F F
62912024-02-08T06:41:50.342Z 21 420-439 57 57 57 92 92 92 F F F
62922024-02-08T06:41:50.342Z 22 440-459 57 57 57 92 92 92 F F F
62932024-02-08T06:41:50.342Z 23 460-479 52 52 52 79 79 79 F F F
62942024-02-08T06:41:50.343Z 24 480-499 56 56 56 90 90 90 F F F
62952024-02-08T06:41:50.343Z 25 500-519 57 57 57 92 92 92 F F F
62962024-02-08T06:41:50.343Z 26 520-539 57 57 57 92 92 92 F F F
62972024-02-08T06:41:50.343Z 27 540-559 57 57 57 92 92 92 F F F
62982024-02-08T06:41:50.343Z 28 560-579 54 54 54 85 85 85 F F F
62992024-02-08T06:41:50.343Z 29 580-599 55 55 55 87 87 87 F F F
63002024-02-08T06:41:50.343ZMax gen: 57, Max flush: 92
63012024-02-08T06:41:50.343Z
63022024-02-08T06:41:50.343ZBegin loop 29
63032024-02-08T06:41:50.343ZDownstairs to restart: 0
63042024-02-08T06:41:50.359Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.600894441Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370}
63052024-02-08T06:41:50.366Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
63062024-02-08T06:41:50.366Z,"time":"2024-02-08T06:41:49.60117821Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370}
63072024-02-08T06:41:50.366Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601255884Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370}
63082024-02-08T06:41:50.366Z{"msg":"Crucible 9a051fb4-b8fd-4233-898e-a2f9262c20eb has session id: 00df90a7-ad8a-4a11-8231-040118dcd51b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.6013119Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63092024-02-08T06:41:50.366Z{"msg":"Upstairs opts: Upstairs UUID: 9a051fb4-b8fd-4233-898e-a2f9262c20eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601364326Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63102024-02-08T06:41:50.367Z{"msg":"Crucible stats registered with UUID: 9a051fb4-b8fd-4233-898e-a2f9262c20eb","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601420062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63112024-02-08T06:41:50.367Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601471688Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370}
63122024-02-08T06:41:50.367Z{"msg":"9a051fb4-b8fd-4233-898e-a2f9262c20eb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601499526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63132024-02-08T06:41:50.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601527954Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63142024-02-08T06:41:50.367Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.60158275Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63152024-02-08T06:41:50.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601631866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63162024-02-08T06:41:50.367Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601677963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63172024-02-08T06:41:50.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601725979Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63182024-02-08T06:41:50.367Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601772195Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63192024-02-08T06:41:50.367Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601806083Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"io task","client":"0","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63202024-02-08T06:41:50.367Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.60184837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"io task","client":"2","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63212024-02-08T06:41:50.367Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601891327Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"io task","client":"1","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63222024-02-08T06:41:50.367Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601918564Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"io task","client":"0","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63232024-02-08T06:41:50.367Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601949852Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"io task","client":"2","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63242024-02-08T06:41:50.367Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.601965781Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"io task","client":"1","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63252024-02-08T06:41:50.372Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.602519399Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63262024-02-08T06:41:50.373Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.602556727Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63272024-02-08T06:41:50.373Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.602573385Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63282024-02-08T06:41:50.373Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.602655029Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63292024-02-08T06:41:50.373Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.603107485Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63302024-02-08T06:41:50.373Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.603245465Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63312024-02-08T06:41:50.373Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.603589459Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63322024-02-08T06:41:50.373Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.603637195Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63332024-02-08T06:41:50.373Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.603772145Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63342024-02-08T06:41:50.373Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.603801403Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63352024-02-08T06:41:50.373Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.603816742Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63362024-02-08T06:41:50.373Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.603829651Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63372024-02-08T06:41:50.373Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.60385027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63382024-02-08T06:41:50.373Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.603864869Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63392024-02-08T06:41:50.373Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604168556Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63402024-02-08T06:41:50.373Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604623201Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63412024-02-08T06:41:50.374Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604658599Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63422024-02-08T06:41:50.374Z{"msg":"[0]R flush_numbers[0..12]: [62, 91, 75, 84, 91, 91, 73, 78, 89, 92, 87, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604676288Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63432024-02-08T06:41:50.374Z{"msg":"[0]R generation[0..12]: [40, 56, 49, 54, 56, 56, 48, 51, 56, 57, 55, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604698236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63442024-02-08T06:41:50.374Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604719254Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63452024-02-08T06:41:50.374Z{"msg":"[1]R flush_numbers[0..12]: [62, 91, 75, 84, 91, 91, 73, 78, 89, 92, 87, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604740683Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63462024-02-08T06:41:50.374Z{"msg":"[1]R generation[0..12]: [40, 56, 49, 54, 56, 56, 48, 51, 56, 57, 55, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604763051Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63472024-02-08T06:41:50.374Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.60478402Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63482024-02-08T06:41:50.374Z{"msg":"[2]R flush_numbers[0..12]: [62, 91, 75, 84, 91, 91, 73, 78, 89, 92, 87, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604799098Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63492024-02-08T06:41:50.374ZWait for a query_work_queue command to finish before sending IO
63502024-02-08T06:41:50.374Z{"msg":"[2]R generation[0..12]: [40, 56, 49, 54, 56, 56, 48, 51, 56, 57, 55, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604824526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63512024-02-08T06:41:50.374Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604840115Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63522024-02-08T06:41:50.374Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604859664Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63532024-02-08T06:41:50.374Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604874263Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63542024-02-08T06:41:50.374Z{"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604894321Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63552024-02-08T06:41:50.374Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.60490797Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63562024-02-08T06:41:50.375Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604927889Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63572024-02-08T06:41:50.375Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604940468Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63582024-02-08T06:41:50.375Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604960106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63592024-02-08T06:41:50.375Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604973295Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63602024-02-08T06:41:50.375Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.604987294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63612024-02-08T06:41:50.375Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.605019682Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63622024-02-08T06:41:50.375Z{"msg":"9a051fb4-b8fd-4233-898e-a2f9262c20eb is now active with session: 00df90a7-ad8a-4a11-8231-040118dcd51b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.605033961Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63632024-02-08T06:41:50.375Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.605053029Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370,"session_id":"00df90a7-ad8a-4a11-8231-040118dcd51b"}
63642024-02-08T06:41:50.375Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:49.605065788Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370}
63652024-02-08T06:41:50.375ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63662024-02-08T06:41:50.375ZLoading write count information from file "/tmp/test_repair_verify.data"
63672024-02-08T06:41:50.375ZRun Repair workload
63682024-02-08T06:41:50.375Z01/30 Write block 33 len 12288 data: 5 4 4
63692024-02-08T06:41:50.375Z02/30 Flush
63702024-02-08T06:41:50.375Z03/30 Write block 381 len 20480 data: 4 5 4 7 6
63712024-02-08T06:41:50.376Z04/30 Write block 309 len 4096 data: 3
63722024-02-08T06:41:50.376Z05/30 Write block 579 len 24576 data: 5 4 3 4 6 6
63732024-02-08T06:41:50.376Z06/30 Write block 91 len 36864 data: 10 11 10 9 8 6 5 4 5
63742024-02-08T06:41:50.376Z07/30 Read block 376 len 4096
63752024-02-08T06:41:50.376Z08/30 Read block 492 len 16384
63762024-02-08T06:41:50.376Z09/30 Read block 578 len 32768
63772024-02-08T06:41:50.379Z10/30 Read block 73 len 4096
63782024-02-08T06:41:50.379Z11/30 Read block 401 len 40960
63792024-02-08T06:41:50.379Z12/30 Read block 417 len 20480
63802024-02-08T06:41:50.380Z13/30 Write block 320 len 32768 data: 9 7 7 7 5 6 4 4
63812024-02-08T06:41:50.380Z14/30 Read block 449 len 20480
63822024-02-08T06:41:50.380Z15/30 Read block 35 len 32768
63832024-02-08T06:41:50.380Z16/30 Write block 12 len 4096 data: 5
63842024-02-08T06:41:50.380Z17/30 Read block 36 len 40960
63852024-02-08T06:41:50.380Z18/30 Read block 186 len 28672
63862024-02-08T06:41:50.383Z19/30 Write block 563 len 20480 data: 4 4 5 5 5
63872024-02-08T06:41:50.383Z20/30 Read block 307 len 16384
63882024-02-08T06:41:50.383Z21/30 Flush
63892024-02-08T06:41:50.386Z22/30 Write block 339 len 32768 data: 8 7 6 5 4 5 6 6
63902024-02-08T06:41:50.387Z23/30 Read block 182 len 16384
63912024-02-08T06:41:50.387Z24/30 Read block 213 len 12288
63922024-02-08T06:41:50.389Z25/30 Write block 426 len 24576 data: 5 3 3 3 4 3
63932024-02-08T06:41:50.389Z26/30 Read block 343 len 16384
63942024-02-08T06:41:50.389Z27/30 Write block 547 len 20480 data: 5 5 7 5 6
63952024-02-08T06:41:50.393Z28/30 Write block 412 len 32768 data: 4 4 5 4 4 5 7 6
63962024-02-08T06:41:50.393Z29/30 Write block 13 len 40960 data: 6 5 4 3 4 5 5 6 6 7
63972024-02-08T06:41:50.393Z30/30 Read block 245 len 8192
63982024-02-08T06:41:50.393Z----------------------------------------------------------------
63992024-02-08T06:41:50.393Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:28
64002024-02-08T06:41:50.393ZStates: Active Active Active
64012024-02-08T06:41:50.393ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
64022024-02-08T06:41:50.393Z 3 Acked 1002 Write 5 Sent Done Done false
64032024-02-08T06:41:50.393Z 4 Acked 1003 Write 1 Sent Done Done false
64042024-02-08T06:41:50.393Z 5 Acked 1004 Write 6 Sent Done Done false
64052024-02-08T06:41:50.393Z 6 Acked 1005 Write 9 Sent Done Done false
64062024-02-08T06:41:50.393Z 7 Acked 1006 Read 1 Sent Done Done false
64072024-02-08T06:41:50.393Z 8 Acked 1007 Read 4 Sent Done Done false
64082024-02-08T06:41:50.394Z 9 Acked 1008 Read 8 Sent Done Done false
64092024-02-08T06:41:50.394Z 10 Acked 1009 Read 1 Sent Done Done false
64102024-02-08T06:41:50.394Z 11 Acked 1010 Read 10 Sent Done Done false
64112024-02-08T06:41:50.394Z 12 Acked 1011 Read 5 Sent Done Done false
64122024-02-08T06:41:50.394Z 13 Acked 1012 Write 8 Sent Done Done false
64132024-02-08T06:41:50.394Z 14 Acked 1013 Read 5 Sent Done Done false
64142024-02-08T06:41:50.394Z 15 Acked 1014 Read 8 Sent Done Done false
64152024-02-08T06:41:50.394Z 16 Acked 1015 Write 1 Sent Done Done false
64162024-02-08T06:41:50.394Z 17 Acked 1016 Read 10 Sent Done Done false
64172024-02-08T06:41:50.394Z 18 Acked 1017 Read 7 Sent Done Done false
64182024-02-08T06:41:50.394Z 19 Acked 1018 Write 5 Sent Done Done false
64192024-02-08T06:41:50.394Z 20 Acked 1019 Read 4 Sent Done Done false
64202024-02-08T06:41:50.394Z 21 Acked 1020 Flush 0 Sent Done Done false
64212024-02-08T06:41:50.394Z 22 Acked 1021 Write 8 Sent Done Done false
64222024-02-08T06:41:50.394Z 23 Acked 1022 Read 4 Sent Done Done false
64232024-02-08T06:41:50.395Z 24 Acked 1023 Read 3 Sent Done Done false
64242024-02-08T06:41:50.395Z 25 Acked 1024 Write 6 Sent Done Done false
64252024-02-08T06:41:50.395Z 26 Acked 1025 Read 4 Sent Done Done false
64262024-02-08T06:41:50.395Z 27 Acked 1026 Write 5 Sent Done Done false
64272024-02-08T06:41:50.395Z 28 Acked 1027 Write 8 Sent Done Done false
64282024-02-08T06:41:50.395Z 29 Acked 1028 Write 10 Sent Done Done false
64292024-02-08T06:41:50.395Z 30 Acked 1029 Read 2 Sent Sent Done false
64302024-02-08T06:41:50.395Z STATES DS:0 DS:1 DS:2 TOTAL
64312024-02-08T06:41:50.395Z New 0 0 0 0
64322024-02-08T06:41:50.395Z Sent 28 1 0 29
64332024-02-08T06:41:50.395Z Done 0 27 28 55
64342024-02-08T06:41:50.395Z Skipped 0 0 0 0
64352024-02-08T06:41:50.395Z Error 0 0 0 0
64362024-02-08T06:41:50.395ZLast Flush: 1001 1020 1020
64372024-02-08T06:41:50.399ZDownstairs last five completed: 1001 1000
64382024-02-08T06:41:50.399ZUpstairs last five completed: 30 29 28 27 26
64392024-02-08T06:41:50.399Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:49.634859631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1370}
64402024-02-08T06:41:50.399ZWrote out file "/tmp/test_repair_verify.data"
64412024-02-08T06:41:50.399Z
64422024-02-08T06:41:50.402ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
64432024-02-08T06:41:50.417ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64442024-02-08T06:41:50.418Z 0 000-019 40 58 58 62 94 94 F T T <---
64452024-02-08T06:41:50.418Z 1 020-039 58 58 58 93 93 93 F T T <---
64462024-02-08T06:41:50.418Z 2 040-059 49 49 49 75 75 75 F F F
64472024-02-08T06:41:50.418Z 3 060-079 54 54 54 84 84 84 F F F
64482024-02-08T06:41:50.418Z 4 080-099 56 58 58 91 94 94 F F F <---
64492024-02-08T06:41:50.421Z 5 100-119 56 56 56 91 91 91 F F F
64502024-02-08T06:41:50.421Z 6 120-139 48 48 48 73 73 73 F F F
64512024-02-08T06:41:50.421Z 7 140-159 51 51 51 78 78 78 F F F
64522024-02-08T06:41:50.421Z 8 160-179 56 56 56 89 89 89 F F F
64532024-02-08T06:41:50.421Z 9 180-199 57 57 57 92 92 92 F F F
64542024-02-08T06:41:50.421Z 10 200-219 55 55 55 87 87 87 F F F
64552024-02-08T06:41:50.421Z 11 220-239 56 56 56 91 91 91 F F F
64562024-02-08T06:41:50.421Z 12 240-259 52 52 52 79 79 79 F F F
64572024-02-08T06:41:50.421Z 13 260-279 56 56 56 89 89 89 F F F
64582024-02-08T06:41:50.421Z 14 280-299 54 54 54 85 85 85 F F F
64592024-02-08T06:41:50.422Z 15 300-319 52 58 58 81 94 94 F F F <---
64602024-02-08T06:41:50.422Z 16 320-339 56 58 58 89 94 94 F T T <---
64612024-02-08T06:41:50.422Z 17 340-359 54 54 54 85 85 85 F T T <---
64622024-02-08T06:41:50.422Z 18 360-379 47 47 47 72 72 72 F F F
64632024-02-08T06:41:50.422Z 19 380-399 55 58 58 87 94 94 F F F <---
64642024-02-08T06:41:50.422Z 20 400-419 55 55 55 87 87 87 F T T <---
64652024-02-08T06:41:50.422Z 21 420-439 57 57 57 92 92 92 F T T <---
64662024-02-08T06:41:50.422Z 22 440-459 57 57 57 92 92 92 F F F
64672024-02-08T06:41:50.422Z 23 460-479 52 52 52 79 79 79 F F F
64682024-02-08T06:41:50.422Z 24 480-499 56 56 56 90 90 90 F F F
64692024-02-08T06:41:50.422Z 25 500-519 57 57 57 92 92 92 F F F
64702024-02-08T06:41:50.422Z 26 520-539 57 57 57 92 92 92 F F F
64712024-02-08T06:41:50.422Z 27 540-559 57 57 57 92 92 92 F T T <---
64722024-02-08T06:41:50.422Z 28 560-579 54 58 58 85 94 94 F F F <---
64732024-02-08T06:41:50.422Z 29 580-599 55 58 58 87 94 94 F F F <---
64742024-02-08T06:41:50.422ZMax gen: 58, Max flush: 94
64752024-02-08T06:41:50.423ZError: Difference in extent metadata found!
64762024-02-08T06:41:50.423ZA Difference in extent metadata is expected here
64772024-02-08T06:41:50.423ZOn loop 29
64782024-02-08T06:41:50.423Z
64792024-02-08T06:41:50.426ZVerifying data now
64802024-02-08T06:41:55.019ZNo change after verify
64812024-02-08T06:41:55.019ZLoop: 29 Downstairs dump after verify (and repair):
64822024-02-08T06:41:55.034ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64832024-02-08T06:41:55.034Z 0 000-019 59 59 59 95 95 95 F F F
64842024-02-08T06:41:55.034Z 1 020-039 59 59 59 95 95 95 F F F
64852024-02-08T06:41:55.034Z 2 040-059 49 49 49 75 75 75 F F F
64862024-02-08T06:41:55.034Z 3 060-079 54 54 54 84 84 84 F F F
64872024-02-08T06:41:55.035Z 4 080-099 58 58 58 94 94 94 F F F
64882024-02-08T06:41:55.035Z 5 100-119 56 56 56 91 91 91 F F F
64892024-02-08T06:41:55.035Z 6 120-139 48 48 48 73 73 73 F F F
64902024-02-08T06:41:55.035Z 7 140-159 51 51 51 78 78 78 F F F
64912024-02-08T06:41:55.035Z 8 160-179 56 56 56 89 89 89 F F F
64922024-02-08T06:41:55.035Z 9 180-199 57 57 57 92 92 92 F F F
64932024-02-08T06:41:55.035Z 10 200-219 55 55 55 87 87 87 F F F
64942024-02-08T06:41:55.035Z 11 220-239 56 56 56 91 91 91 F F F
64952024-02-08T06:41:55.039Z 12 240-259 52 52 52 79 79 79 F F F
64962024-02-08T06:41:55.039Z 13 260-279 56 56 56 89 89 89 F F F
64972024-02-08T06:41:55.039Z 14 280-299 54 54 54 85 85 85 F F F
64982024-02-08T06:41:55.039Z 15 300-319 58 58 58 94 94 94 F F F
64992024-02-08T06:41:55.040Z 16 320-339 59 59 59 95 95 95 F F F
65002024-02-08T06:41:55.040Z 17 340-359 59 59 59 95 95 95 F F F
65012024-02-08T06:41:55.040Z 18 360-379 47 47 47 72 72 72 F F F
65022024-02-08T06:41:55.040Z 19 380-399 58 58 58 94 94 94 F F F
65032024-02-08T06:41:55.040Z 20 400-419 59 59 59 95 95 95 F F F
65042024-02-08T06:41:55.040Z 21 420-439 59 59 59 95 95 95 F F F
65052024-02-08T06:41:55.040Z 22 440-459 57 57 57 92 92 92 F F F
65062024-02-08T06:41:55.040Z 23 460-479 52 52 52 79 79 79 F F F
65072024-02-08T06:41:55.040Z 24 480-499 56 56 56 90 90 90 F F F
65082024-02-08T06:41:55.041Z 25 500-519 57 57 57 92 92 92 F F F
65092024-02-08T06:41:55.041Z 26 520-539 57 57 57 92 92 92 F F F
65102024-02-08T06:41:55.041Z 27 540-559 59 59 59 95 95 95 F F F
65112024-02-08T06:41:55.041Z 28 560-579 58 58 58 94 94 94 F F F
65122024-02-08T06:41:55.041Z 29 580-599 58 58 58 94 94 94 F F F
65132024-02-08T06:41:55.041ZMax gen: 59, Max flush: 95
65142024-02-08T06:41:55.041Z
65152024-02-08T06:41:55.041ZBegin loop 30
65162024-02-08T06:41:55.042ZDownstairs to restart: 0
65172024-02-08T06:41:55.055Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.295827905Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378}
65182024-02-08T06:41:55.063Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.29616589Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378}
65192024-02-08T06:41:55.063Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296217636Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378}
65202024-02-08T06:41:55.063ZCrucible runtime is spawned
65212024-02-08T06:41:55.063Z{"msg":"Crucible b924779f-42fa-4768-8b03-35762f97e524 has session id: 4652aceb-291c-429d-9949-a1bdae19688c","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296270162Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65222024-02-08T06:41:55.063Z{"msg":"Upstairs opts: Upstairs UUID: b924779f-42fa-4768-8b03-35762f97e524, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296311959Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65232024-02-08T06:41:55.063Z{"msg":"Crucible stats registered with UUID: b924779f-42fa-4768-8b03-35762f97e524","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296356755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65242024-02-08T06:41:55.063Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296398952Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378}
65252024-02-08T06:41:55.063Z{"msg":"b924779f-42fa-4768-8b03-35762f97e524 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296477086Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65262024-02-08T06:41:55.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296515714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65272024-02-08T06:41:55.063Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.29655949Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65282024-02-08T06:41:55.064Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296607197Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65292024-02-08T06:41:55.064Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296638404Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65302024-02-08T06:41:55.064Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296674212Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65312024-02-08T06:41:55.064Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.29670083Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65322024-02-08T06:41:55.064Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296747056Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"io task","client":"2","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65332024-02-08T06:41:55.064Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296774214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"io task","client":"0","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65342024-02-08T06:41:55.064Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296808292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"io task","client":"1","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65352024-02-08T06:41:55.064Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.29682982Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"io task","client":"0","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65362024-02-08T06:41:55.064Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296853488Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"io task","client":"1","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65372024-02-08T06:41:55.064Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.296877686Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"io task","client":"2","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65382024-02-08T06:41:55.064Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.297389988Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65392024-02-08T06:41:55.064Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.297432855Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65402024-02-08T06:41:55.065Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.297480811Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65412024-02-08T06:41:55.065Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.297524328Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65422024-02-08T06:41:55.065Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.298064337Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65432024-02-08T06:41:55.065Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.298094945Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65442024-02-08T06:41:55.065Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.298716908Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65452024-02-08T06:41:55.065Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.298758775Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65462024-02-08T06:41:55.065Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.298775254Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65472024-02-08T06:41:55.065Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.298794083Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65482024-02-08T06:41:55.065Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.298816271Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65492024-02-08T06:41:55.066Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.29883128Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65502024-02-08T06:41:55.066Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.298851458Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65512024-02-08T06:41:55.066Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.298865597Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65522024-02-08T06:41:55.066Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.299412386Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65532024-02-08T06:41:55.071Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.29989992Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65542024-02-08T06:41:55.071Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.299933497Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65552024-02-08T06:41:55.071Z{"msg":"[0]R flush_numbers[0..12]: [95, 95, 75, 84, 94, 91, 73, 78, 89, 92, 87, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.299949736Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65562024-02-08T06:41:55.071Z{"msg":"[0]R generation[0..12]: [59, 59, 49, 54, 58, 56, 48, 51, 56, 57, 55, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.299963885Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65572024-02-08T06:41:55.071Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.299986453Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65582024-02-08T06:41:55.071Z{"msg":"[1]R flush_numbers[0..12]: [95, 95, 75, 84, 94, 91, 73, 78, 89, 92, 87, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300001462Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65592024-02-08T06:41:55.071Z{"msg":"[1]R generation[0..12]: [59, 59, 49, 54, 58, 56, 48, 51, 56, 57, 55, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300015321Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65602024-02-08T06:41:55.072Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300038009Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65612024-02-08T06:41:55.072Z{"msg":"[2]R flush_numbers[0..12]: [95, 95, 75, 84, 94, 91, 73, 78, 89, 92, 87, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300053008Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65622024-02-08T06:41:55.072Z{"msg":"[2]R generation[0..12]: [59, 59, 49, 54, 58, 56, 48, 51, 56, 57, 55, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300095545Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65632024-02-08T06:41:55.072ZWait for a query_work_queue command to finish before sending IO
65642024-02-08T06:41:55.072Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300113013Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65652024-02-08T06:41:55.072Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300128042Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65662024-02-08T06:41:55.072Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300151131Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65672024-02-08T06:41:55.072Z{"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300188048Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65682024-02-08T06:41:55.072Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300210156Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65692024-02-08T06:41:55.073Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300229695Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65702024-02-08T06:41:55.073Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300243094Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65712024-02-08T06:41:55.073Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300271012Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65722024-02-08T06:41:55.073Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300303929Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65732024-02-08T06:41:55.073Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300329167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65742024-02-08T06:41:55.073Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300355795Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65752024-02-08T06:41:55.073Z{"msg":"b924779f-42fa-4768-8b03-35762f97e524 is now active with session: 4652aceb-291c-429d-9949-a1bdae19688c","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300380044Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65762024-02-08T06:41:55.073Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300412621Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378,"session_id":"4652aceb-291c-429d-9949-a1bdae19688c"}
65772024-02-08T06:41:55.074Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.300435299Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378}
65782024-02-08T06:41:55.074ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65792024-02-08T06:41:55.074ZLoading write count information from file "/tmp/test_repair_verify.data"
65802024-02-08T06:41:55.074ZRun Repair workload
65812024-02-08T06:41:55.074Z01/30 Write block 275 len 12288 data: 5 5 5
65822024-02-08T06:41:55.074Z02/30 Flush
65832024-02-08T06:41:55.074Z03/30 Write block 593 len 20480 data: 5 5 3 3 2
65842024-02-08T06:41:55.074Z04/30 Read block 533 len 16384
65852024-02-08T06:41:55.074Z05/30 Write block 437 len 36864 data: 9 7 5 7 6 6 7 5 5
65862024-02-08T06:41:55.075Z06/30 Read block 347 len 16384
65872024-02-08T06:41:55.075Z07/30 Read block 81 len 8192
65882024-02-08T06:41:55.075Z08/30 Write block 398 len 8192 data: 4 5
65892024-02-08T06:41:55.075Z09/30 Write block 116 len 16384 data: 5 4 2 2
65902024-02-08T06:41:55.075Z10/30 Write block 434 len 20480 data: 6 6 9 10 8
65912024-02-08T06:41:55.075Z11/30 Write block 331 len 12288 data: 4 5 6
65922024-02-08T06:41:55.075Z12/30 Write block 499 len 24576 data: 4 4 4 4 2 5
65932024-02-08T06:41:55.075Z13/30 Read block 374 len 40960
65942024-02-08T06:41:55.079Z14/30 Read block 513 len 4096
65952024-02-08T06:41:55.079Z15/30 Read block 430 len 8192
65962024-02-08T06:41:55.080Z16/30 Read block 110 len 4096
65972024-02-08T06:41:55.080Z17/30 Write block 103 len 8192 data: 6 7
65982024-02-08T06:41:55.080Z18/30 Read block 482 len 36864
65992024-02-08T06:41:55.080Z19/30 Read block 434 len 40960
66002024-02-08T06:41:55.080Z20/30 Read block 107 len 8192
66012024-02-08T06:41:55.080Z21/30 Read block 341 len 12288
66022024-02-08T06:41:55.080Z22/30 Read block 34 len 28672
66032024-02-08T06:41:55.080Z23/30 Read block 353 len 8192
66042024-02-08T06:41:55.081Z24/30 Write block 195 len 32768 data: 7 6 8 9 9 8 7 7
66052024-02-08T06:41:55.081Z25/30 Write block 459 len 4096 data: 5
66062024-02-08T06:41:55.081Z26/30 Flush
66072024-02-08T06:41:55.084Z27/30 Write block 210 len 32768 data: 4 6 8 8 8 5 5 7
66082024-02-08T06:41:55.084Z28/30 Read block 532 len 36864
66092024-02-08T06:41:55.085Z29/30 Read block 591 len 20480
66102024-02-08T06:41:55.089Z30/30 Read block 94 len 8192
66112024-02-08T06:41:55.089Z----------------------------------------------------------------
66122024-02-08T06:41:55.089Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
66132024-02-08T06:41:55.089ZStates: Active Active Active
66142024-02-08T06:41:55.090ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
66152024-02-08T06:41:55.090Z 1 Acked 1000 Write 3 Sent Done Done false
66162024-02-08T06:41:55.090Z 2 Acked 1001 Flush 0 Sent Done Done false
66172024-02-08T06:41:55.090Z 3 Acked 1002 Write 5 Sent Done Done false
66182024-02-08T06:41:55.090Z 4 Acked 1003 Read 4 Sent Done Done false
66192024-02-08T06:41:55.090Z 5 Acked 1004 Write 9 Sent Done Done false
66202024-02-08T06:41:55.090Z 6 Acked 1005 Read 4 Sent Done Done false
66212024-02-08T06:41:55.090Z 7 Acked 1006 Read 2 Sent Done Done false
66222024-02-08T06:41:55.091Z 8 Acked 1007 Write 2 Sent Done Done false
66232024-02-08T06:41:55.091Z 9 Acked 1008 Write 4 Sent Done Done false
66242024-02-08T06:41:55.091Z 10 Acked 1009 Write 5 Sent Done Done false
66252024-02-08T06:41:55.091Z 11 Acked 1010 Write 3 Sent Done Done false
66262024-02-08T06:41:55.091Z 12 Acked 1011 Write 6 Sent Done Done false
66272024-02-08T06:41:55.091Z 13 Acked 1012 Read 10 Sent Done Done false
66282024-02-08T06:41:55.091Z 14 Acked 1013 Read 1 Sent Done Done false
66292024-02-08T06:41:55.091Z 15 Acked 1014 Read 2 Sent Done Done false
66302024-02-08T06:41:55.091Z 16 Acked 1015 Read 1 Sent Done Done false
66312024-02-08T06:41:55.092Z 17 Acked 1016 Write 2 Sent Done Done false
66322024-02-08T06:41:55.092Z 18 Acked 1017 Read 9 Sent Done Done false
66332024-02-08T06:41:55.092Z 19 Acked 1018 Read 10 Sent Done Done false
66342024-02-08T06:41:55.092Z 20 Acked 1019 Read 2 Sent Done Done false
66352024-02-08T06:41:55.092Z 21 Acked 1020 Read 3 Sent Done Done false
66362024-02-08T06:41:55.092Z 22 Acked 1021 Read 7 Sent Done Done false
66372024-02-08T06:41:55.092Z 23 Acked 1022 Read 2 Sent Done Done false
66382024-02-08T06:41:55.092Z 24 Acked 1023 Write 8 Sent Done Done false
66392024-02-08T06:41:55.092Z 25 Acked 1024 Write 1 Sent Done Done false
66402024-02-08T06:41:55.093Z 26 Acked 1025 Flush 0 Sent Done Done false
66412024-02-08T06:41:55.093Z 27 Acked 1026 Write 8 Sent Done Done false
66422024-02-08T06:41:55.093Z 28 Acked 1027 Read 9 Sent Done Done false
66432024-02-08T06:41:55.093Z 29 Acked 1028 Read 5 Sent Done Done false
66442024-02-08T06:41:55.093Z 30 Acked 1029 Read 2 Sent Sent Done false
66452024-02-08T06:41:55.093Z STATES DS:0 DS:1 DS:2 TOTAL
66462024-02-08T06:41:55.093Z New 0 0 0 0
66472024-02-08T06:41:55.093Z Sent 30 1 0 31
66482024-02-08T06:41:55.093Z Done 0 29 30 59
66492024-02-08T06:41:55.094Z Skipped 0 0 0 0
66502024-02-08T06:41:55.094Z Error 0 0 0 0
66512024-02-08T06:41:55.094ZLast Flush: 0 1025 1025
66522024-02-08T06:41:55.094ZDownstairs last five completed:
66532024-02-08T06:41:55.094ZUpstairs last five completed: 30 29 28 27 26
66542024-02-08T06:41:55.094Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:54.3275973Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1378}
66552024-02-08T06:41:55.094ZWrote out file "/tmp/test_repair_verify.data"
66562024-02-08T06:41:55.099Z
66572024-02-08T06:41:55.099ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66582024-02-08T06:41:55.108ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66592024-02-08T06:41:55.108Z 0 000-019 59 59 59 95 95 95 F F F
66602024-02-08T06:41:55.108Z 1 020-039 59 59 59 95 95 95 F F F
66612024-02-08T06:41:55.108Z 2 040-059 49 49 49 75 75 75 F F F
66622024-02-08T06:41:55.108Z 3 060-079 54 54 54 84 84 84 F F F
66632024-02-08T06:41:55.108Z 4 080-099 58 58 58 94 94 94 F F F
66642024-02-08T06:41:55.109Z 5 100-119 56 60 60 91 97 97 F F F <---
66652024-02-08T06:41:55.109Z 6 120-139 48 48 48 73 73 73 F F F
66662024-02-08T06:41:55.109Z 7 140-159 51 51 51 78 78 78 F F F
66672024-02-08T06:41:55.109Z 8 160-179 56 56 56 89 89 89 F F F
66682024-02-08T06:41:55.109Z 9 180-199 57 60 60 92 97 97 F F F <---
66692024-02-08T06:41:55.112Z 10 200-219 55 60 60 87 97 97 F T T <---
66702024-02-08T06:41:55.112Z 11 220-239 56 56 56 91 91 91 F F F
66712024-02-08T06:41:55.112Z 12 240-259 52 52 52 79 79 79 F F F
66722024-02-08T06:41:55.112Z 13 260-279 56 60 60 89 96 96 F F F <---
66732024-02-08T06:41:55.112Z 14 280-299 54 54 54 85 85 85 F F F
66742024-02-08T06:41:55.113Z 15 300-319 58 58 58 94 94 94 F F F
66752024-02-08T06:41:55.113Z 16 320-339 59 60 60 95 97 97 F F F <---
66762024-02-08T06:41:55.113Z 17 340-359 59 59 59 95 95 95 F F F
66772024-02-08T06:41:55.113Z 18 360-379 47 47 47 72 72 72 F F F
66782024-02-08T06:41:55.113Z 19 380-399 58 60 60 94 97 97 F F F <---
66792024-02-08T06:41:55.113Z 20 400-419 59 59 59 95 95 95 F F F
66802024-02-08T06:41:55.113Z 21 420-439 59 60 60 95 97 97 F F F <---
66812024-02-08T06:41:55.113Z 22 440-459 57 60 60 92 97 97 F F F <---
66822024-02-08T06:41:55.113Z 23 460-479 52 52 52 79 79 79 F F F
66832024-02-08T06:41:55.113Z 24 480-499 56 60 60 90 97 97 F F F <---
66842024-02-08T06:41:55.113Z 25 500-519 57 60 60 92 97 97 F F F <---
66852024-02-08T06:41:55.113Z 26 520-539 57 57 57 92 92 92 F F F
66862024-02-08T06:41:55.113Z 27 540-559 59 59 59 95 95 95 F F F
66872024-02-08T06:41:55.113Z 28 560-579 58 58 58 94 94 94 F F F
66882024-02-08T06:41:55.114Z 29 580-599 58 60 60 94 97 97 F F F <---
66892024-02-08T06:41:55.114ZMax gen: 60, Max flush: 97
66902024-02-08T06:41:55.114ZError: Difference in extent metadata found!
66912024-02-08T06:41:55.114ZA Difference in extent metadata is expected here
66922024-02-08T06:41:55.114ZOn loop 30
66932024-02-08T06:41:55.114Z
66942024-02-08T06:41:55.117ZVerifying data now
66952024-02-08T06:41:55.639ZNo change after verify
66962024-02-08T06:41:55.639ZLoop: 30 Downstairs dump after verify (and repair):
66972024-02-08T06:41:55.655ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66982024-02-08T06:41:55.655Z 0 000-019 59 59 59 95 95 95 F F F
66992024-02-08T06:41:55.655Z 1 020-039 59 59 59 95 95 95 F F F
67002024-02-08T06:41:55.655Z 2 040-059 49 49 49 75 75 75 F F F
67012024-02-08T06:41:55.655Z 3 060-079 54 54 54 84 84 84 F F F
67022024-02-08T06:41:55.655Z 4 080-099 58 58 58 94 94 94 F F F
67032024-02-08T06:41:55.655Z 5 100-119 60 60 60 97 97 97 F F F
67042024-02-08T06:41:55.655Z 6 120-139 48 48 48 73 73 73 F F F
67052024-02-08T06:41:55.658Z 7 140-159 51 51 51 78 78 78 F F F
67062024-02-08T06:41:55.658Z 8 160-179 56 56 56 89 89 89 F F F
67072024-02-08T06:41:55.658Z 9 180-199 60 60 60 97 97 97 F F F
67082024-02-08T06:41:55.659Z 10 200-219 61 61 61 98 98 98 F F F
67092024-02-08T06:41:55.659Z 11 220-239 56 56 56 91 91 91 F F F
67102024-02-08T06:41:55.659Z 12 240-259 52 52 52 79 79 79 F F F
67112024-02-08T06:41:55.659Z 13 260-279 60 60 60 96 96 96 F F F
67122024-02-08T06:41:55.659Z 14 280-299 54 54 54 85 85 85 F F F
67132024-02-08T06:41:55.659Z 15 300-319 58 58 58 94 94 94 F F F
67142024-02-08T06:41:55.659Z 16 320-339 60 60 60 97 97 97 F F F
67152024-02-08T06:41:55.659Z 17 340-359 59 59 59 95 95 95 F F F
67162024-02-08T06:41:55.659Z 18 360-379 47 47 47 72 72 72 F F F
67172024-02-08T06:41:55.659Z 19 380-399 60 60 60 97 97 97 F F F
67182024-02-08T06:41:55.659Z 20 400-419 59 59 59 95 95 95 F F F
67192024-02-08T06:41:55.659Z 21 420-439 60 60 60 97 97 97 F F F
67202024-02-08T06:41:55.659Z 22 440-459 60 60 60 97 97 97 F F F
67212024-02-08T06:41:55.659Z 23 460-479 52 52 52 79 79 79 F F F
67222024-02-08T06:41:55.660Z 24 480-499 60 60 60 97 97 97 F F F
67232024-02-08T06:41:55.660Z 25 500-519 60 60 60 97 97 97 F F F
67242024-02-08T06:41:55.660Z 26 520-539 57 57 57 92 92 92 F F F
67252024-02-08T06:41:55.660Z 27 540-559 59 59 59 95 95 95 F F F
67262024-02-08T06:41:55.660Z 28 560-579 58 58 58 94 94 94 F F F
67272024-02-08T06:41:55.660Z 29 580-599 60 60 60 97 97 97 F F F
67282024-02-08T06:41:55.660ZMax gen: 61, Max flush: 98
67292024-02-08T06:41:55.660Z
67302024-02-08T06:41:55.660ZBegin loop 31
67312024-02-08T06:41:55.660ZDownstairs to restart: 1
67322024-02-08T06:41:55.676Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917262789Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386}
67332024-02-08T06:41:55.676Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917520329Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386}
67342024-02-08T06:41:55.682Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
67352024-02-08T06:41:55.682Z,"time":"2024-02-08T06:41:54.917555567Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386}
67362024-02-08T06:41:55.682Z{"msg":"Crucible 425e22bc-2d3d-4d3b-858f-5454005cbf0a has session id: 81fa85de-489e-4ad8-9e3d-928b33203b32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917601223Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67372024-02-08T06:41:55.682Z{"msg":"Upstairs opts: Upstairs UUID: 425e22bc-2d3d-4d3b-858f-5454005cbf0a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.9176454Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67382024-02-08T06:41:55.683Z{"msg":"Crucible stats registered with UUID: 425e22bc-2d3d-4d3b-858f-5454005cbf0a","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917671148Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67392024-02-08T06:41:55.683Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917691857Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386}
67402024-02-08T06:41:55.683Z{"msg":"425e22bc-2d3d-4d3b-858f-5454005cbf0a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917795229Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67412024-02-08T06:41:55.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917823027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67422024-02-08T06:41:55.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917844525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67432024-02-08T06:41:55.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917866863Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67442024-02-08T06:41:55.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917881652Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67452024-02-08T06:41:55.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.91790467Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67462024-02-08T06:41:55.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917935088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67472024-02-08T06:41:55.683Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917960516Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"io task","client":"2","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67482024-02-08T06:41:55.683Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.917997583Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"io task","client":"0","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67492024-02-08T06:41:55.683Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.918024261Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"io task","client":"1","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67502024-02-08T06:41:55.683Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.918122894Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"io task","client":"1","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67512024-02-08T06:41:55.683Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.918161041Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"io task","client":"2","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67522024-02-08T06:41:55.683Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.918188449Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"io task","client":"0","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67532024-02-08T06:41:55.684Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.91885043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67542024-02-08T06:41:55.684Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.918902086Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67552024-02-08T06:41:55.684Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.918929553Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67562024-02-08T06:41:55.684Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.918961871Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67572024-02-08T06:41:55.684Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.919591664Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67582024-02-08T06:41:55.684Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.91964492Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67592024-02-08T06:41:55.684Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.919894271Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67602024-02-08T06:41:55.684Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.919942477Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67612024-02-08T06:41:55.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.920156881Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67622024-02-08T06:41:55.684Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.920204238Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67632024-02-08T06:41:55.684Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.920231316Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67642024-02-08T06:41:55.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.920300371Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67652024-02-08T06:41:55.684Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.920335418Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67662024-02-08T06:41:55.684Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.920355986Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67672024-02-08T06:41:55.684Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.920524984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67682024-02-08T06:41:55.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.920969191Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67692024-02-08T06:41:55.685Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921007828Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67702024-02-08T06:41:55.685Z{"msg":"[0]R flush_numbers[0..12]: [95, 95, 75, 84, 94, 97, 73, 78, 89, 97, 98, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921025866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67712024-02-08T06:41:55.685Z{"msg":"[0]R generation[0..12]: [59, 59, 49, 54, 58, 60, 48, 51, 56, 60, 61, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921048354Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67722024-02-08T06:41:55.685Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921063133Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67732024-02-08T06:41:55.685Z{"msg":"[1]R flush_numbers[0..12]: [95, 95, 75, 84, 94, 97, 73, 78, 89, 97, 98, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921083872Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67742024-02-08T06:41:55.685Z{"msg":"[1]R generation[0..12]: [59, 59, 49, 54, 58, 60, 48, 51, 56, 60, 61, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921098181Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67752024-02-08T06:41:55.685Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921119129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67762024-02-08T06:41:55.685Z{"msg":"[2]R flush_numbers[0..12]: [95, 95, 75, 84, 94, 97, 73, 78, 89, 97, 98, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921133228Z","hostname":"Wait for a query_work_queue command to finish before sending IO
67772024-02-08T06:41:55.685Zip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67782024-02-08T06:41:55.685Z{"msg":"[2]R generation[0..12]: [59, 59, 49, 54, 58, 60, 48, 51, 56, 60, 61, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921159436Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67792024-02-08T06:41:55.685Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921174125Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67802024-02-08T06:41:55.685Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921194263Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67812024-02-08T06:41:55.685Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921208093Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67822024-02-08T06:41:55.685Z{"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921227941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67832024-02-08T06:41:55.685Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.92124173Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67842024-02-08T06:41:55.686Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921263178Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67852024-02-08T06:41:55.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921283837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67862024-02-08T06:41:55.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921311575Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67872024-02-08T06:41:55.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921326284Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67882024-02-08T06:41:55.686Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921340233Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67892024-02-08T06:41:55.686Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921353172Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67902024-02-08T06:41:55.686Z{"msg":"425e22bc-2d3d-4d3b-858f-5454005cbf0a is now active with session: 81fa85de-489e-4ad8-9e3d-928b33203b32","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.92137319Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67912024-02-08T06:41:55.686Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921385779Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386,"session_id":"81fa85de-489e-4ad8-9e3d-928b33203b32"}
67922024-02-08T06:41:55.686Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:54.921405428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386}
67932024-02-08T06:41:55.686ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67942024-02-08T06:41:55.686ZLoading write count information from file "/tmp/test_repair_verify.data"
67952024-02-08T06:41:55.686ZRun Repair workload
67962024-02-08T06:41:55.686Z01/30 Write block 331 len 16384 data: 5 6 7 8
67972024-02-08T06:41:55.690Z02/30 Read block 475 len 4096
67982024-02-08T06:41:55.690Z03/30 Flush
67992024-02-08T06:41:55.690Z04/30 Flush
68002024-02-08T06:41:55.690Z05/30 Write block 265 len 24576 data: 5 5 6 8 8 6
68012024-02-08T06:41:55.690Z06/30 Flush
68022024-02-08T06:41:55.690Z07/30 Write block 535 len 12288 data: 8 9 8
68032024-02-08T06:41:55.690Z08/30 Write block 31 len 32768 data: 4 4 6 5 5 4 4 5
68042024-02-08T06:41:55.690Z09/30 Read block 543 len 32768
68052024-02-08T06:41:55.693Z10/30 Write block 579 len 36864 data: 6 5 4 5 7 7 4 4 5
68062024-02-08T06:41:55.693Z11/30 Read block 512 len 12288
68072024-02-08T06:41:55.694Z12/30 Read block 537 len 20480
68082024-02-08T06:41:55.697Z13/30 Read block 78 len 8192
68092024-02-08T06:41:55.697Z14/30 Write block 100 len 8192 data: 6 6
68102024-02-08T06:41:55.697Z15/30 Read block 290 len 32768
68112024-02-08T06:41:55.697Z16/30 Read block 533 len 12288
68122024-02-08T06:41:55.697Z17/30 Read block 143 len 16384
68132024-02-08T06:41:55.700Z18/30 Read block 524 len 16384
68142024-02-08T06:41:55.700Z19/30 Write block 459 len 24576 data: 6 4 5 6 6 3
68152024-02-08T06:41:55.700Z20/30 Read block 91 len 36864
68162024-02-08T06:41:55.703Z21/30 Write block 590 len 32768 data: 7 6 5 6 6 4 4 3
68172024-02-08T06:41:55.703Z22/30 Read block 318 len 8192
68182024-02-08T06:41:55.703Z23/30 Read block 331 len 12288
68192024-02-08T06:41:55.703Z24/30 Read block 79 len 40960
68202024-02-08T06:41:55.716Z25/30 Write block 165 len 20480 data: 7 7 7 6 7
68212024-02-08T06:41:55.716Z26/30 Write block 573 len 40960 data: 4 4 4 4 4 4 7 6 5 6
68222024-02-08T06:41:55.716Z27/30 Read block 423 len 12288
68232024-02-08T06:41:55.716Z28/30 Read block 95 len 20480
68242024-02-08T06:41:55.716Z29/30 Write block 301 len 12288 data: 8 8 7
68252024-02-08T06:41:55.716Z30/30 Write block 338 len 36864 data: 9 9 8 7 6 5 6 7 7
68262024-02-08T06:41:55.716Z----------------------------------------------------------------
68272024-02-08T06:41:55.716Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
68282024-02-08T06:41:55.717ZStates: Active Active Active
68292024-02-08T06:41:55.717ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
68302024-02-08T06:41:55.717Z 1 Acked 1000 Write 4 Done Sent Done false
68312024-02-08T06:41:55.717Z 2 Acked 1001 Read 1 Done Sent Done false
68322024-02-08T06:41:55.717Z 3 Acked 1002 Flush 0 Done Sent Done false
68332024-02-08T06:41:55.717Z 4 Acked 1003 Flush 0 Done Sent Done false
68342024-02-08T06:41:55.717Z 5 Acked 1004 Write 6 Done Sent Done false
68352024-02-08T06:41:55.717Z 6 Acked 1005 Flush 0 Done Sent Done false
68362024-02-08T06:41:55.717Z 7 Acked 1006 Write 3 Done Sent Done false
68372024-02-08T06:41:55.717Z 8 Acked 1007 Write 8 Done Sent Done false
68382024-02-08T06:41:55.717Z 9 Acked 1008 Read 8 Done Sent Done false
68392024-02-08T06:41:55.717Z 10 Acked 1009 Write 9 Done Sent Done false
68402024-02-08T06:41:55.717Z 11 Acked 1010 Read 3 Done Sent Done false
68412024-02-08T06:41:55.717Z 12 Acked 1011 Read 5 Done Sent Done false
68422024-02-08T06:41:55.717Z 13 Acked 1012 Read 2 Done Sent Done false
68432024-02-08T06:41:55.717Z 14 Acked 1013 Write 2 Done Sent Done false
68442024-02-08T06:41:55.718Z 15 Acked 1014 Read 8 Done Sent Done false
68452024-02-08T06:41:55.718Z 16 Acked 1015 Read 3 Done Sent Done false
68462024-02-08T06:41:55.718Z 17 Acked 1016 Read 4 Done Sent Done false
68472024-02-08T06:41:55.718Z 18 Acked 1017 Read 4 Done Sent Done false
68482024-02-08T06:41:55.718Z 19 Acked 1018 Write 6 Done Sent Done false
68492024-02-08T06:41:55.718Z 20 Acked 1019 Read 9 Done Sent Done false
68502024-02-08T06:41:55.718Z 21 Acked 1020 Write 8 Done Sent Done false
68512024-02-08T06:41:55.718Z 22 Acked 1021 Read 2 Done Sent Done false
68522024-02-08T06:41:55.718Z 23 Acked 1022 Read 3 Done Sent Done false
68532024-02-08T06:41:55.718Z 24 Acked 1023 Read 10 Done Sent Done false
68542024-02-08T06:41:55.718Z 25 Acked 1024 Write 5 Done Sent Done false
68552024-02-08T06:41:55.718Z 26 Acked 1025 Write 10 Done Sent Done false
68562024-02-08T06:41:55.718Z 27 Acked 1026 Read 3 Done Sent Done false
68572024-02-08T06:41:55.718Z 28 Acked 1027 Read 5 Done Sent Done false
68582024-02-08T06:41:55.719Z 29 Acked 1028 Write 3 Done Sent Done false
68592024-02-08T06:41:55.719Z 30 Acked 1029 Write 9 Sent Sent Sent false
68602024-02-08T06:41:55.719Z STATES DS:0 DS:1 DS:2 TOTAL
68612024-02-08T06:41:55.719Z New 0 0 0 0
68622024-02-08T06:41:55.719Z Sent 1 30 1 32
68632024-02-08T06:41:55.719Z Done 29 0 29 58
68642024-02-08T06:41:55.719Z Skipped 0 0 0 0
68652024-02-08T06:41:55.719Z Error 0 0 0 0
68662024-02-08T06:41:55.719ZLast Flush: 1005 0 1005
68672024-02-08T06:41:55.719ZDownstairs last five completed:
68682024-02-08T06:41:55.719ZUpstairs last five completed: 30 29 28 27 26
68692024-02-08T06:41:55.719Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:54.950862196Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1386}
68702024-02-08T06:41:55.719ZWrote out file "/tmp/test_repair_verify.data"
68712024-02-08T06:41:55.719Z
68722024-02-08T06:41:55.719ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68732024-02-08T06:41:55.734ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68742024-02-08T06:41:55.734Z 0 000-019 59 59 59 95 95 95 F F F
68752024-02-08T06:41:55.734Z 1 020-039 59 59 59 95 95 95 T F T <---
68762024-02-08T06:41:55.734Z 2 040-059 49 49 49 75 75 75 F F F
68772024-02-08T06:41:55.734Z 3 060-079 54 54 54 84 84 84 F F F
68782024-02-08T06:41:55.734Z 4 080-099 58 58 58 94 94 94 F F F
68792024-02-08T06:41:55.737Z 5 100-119 60 60 60 97 97 97 T F T <---
68802024-02-08T06:41:55.738Z 6 120-139 48 48 48 73 73 73 F F F
68812024-02-08T06:41:55.738Z 7 140-159 51 51 51 78 78 78 F F F
68822024-02-08T06:41:55.738Z 8 160-179 56 56 56 89 89 89 T F T <---
68832024-02-08T06:41:55.738Z 9 180-199 60 60 60 97 97 97 F F F
68842024-02-08T06:41:55.738Z 10 200-219 61 61 61 98 98 98 F F F
68852024-02-08T06:41:55.738Z 11 220-239 56 56 56 91 91 91 F F F
68862024-02-08T06:41:55.738Z 12 240-259 52 52 52 79 79 79 F F F
68872024-02-08T06:41:55.738Z 13 260-279 62 60 62 101 96 101 F F F <---
68882024-02-08T06:41:55.738Z 14 280-299 54 54 54 85 85 85 F F F
68892024-02-08T06:41:55.738Z 15 300-319 58 58 58 94 94 94 T F T <---
68902024-02-08T06:41:55.738Z 16 320-339 62 60 62 99 97 99 T F T <---
68912024-02-08T06:41:55.738Z 17 340-359 59 59 59 95 95 95 T F T <---
68922024-02-08T06:41:55.738Z 18 360-379 47 47 47 72 72 72 F F F
68932024-02-08T06:41:55.738Z 19 380-399 60 60 60 97 97 97 F F F
68942024-02-08T06:41:55.739Z 20 400-419 59 59 59 95 95 95 F F F
68952024-02-08T06:41:55.739Z 21 420-439 60 60 60 97 97 97 F F F
68962024-02-08T06:41:55.739Z 22 440-459 60 60 60 97 97 97 T F T <---
68972024-02-08T06:41:55.739Z 23 460-479 52 52 52 79 79 79 T F T <---
68982024-02-08T06:41:55.739Z 24 480-499 60 60 60 97 97 97 F F F
68992024-02-08T06:41:55.739Z 25 500-519 60 60 60 97 97 97 F F F
69002024-02-08T06:41:55.739Z 26 520-539 57 57 57 92 92 92 T F T <---
69012024-02-08T06:41:55.739Z 27 540-559 59 59 59 95 95 95 F F F
69022024-02-08T06:41:55.739Z 28 560-579 58 58 58 94 94 94 T F T <---
69032024-02-08T06:41:55.739Z 29 580-599 60 60 60 97 97 97 T F T <---
69042024-02-08T06:41:55.739ZMax gen: 62, Max flush: 101
69052024-02-08T06:41:55.739ZError: Difference in extent metadata found!
69062024-02-08T06:41:55.739ZA Difference in extent metadata is expected here
69072024-02-08T06:41:55.739ZOn loop 31
69082024-02-08T06:41:55.740Z
69092024-02-08T06:41:55.743ZVerifying data now
69102024-02-08T06:42:00.335ZNo change after verify
69112024-02-08T06:42:00.335ZLoop: 31 Downstairs dump after verify (and repair):
69122024-02-08T06:42:00.350ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
69132024-02-08T06:42:00.350Z 0 000-019 59 59 59 95 95 95 F F F
69142024-02-08T06:42:00.356Z 1 020-039 63 63 63 102 102 102 F F F
69152024-02-08T06:42:00.357Z 2 040-059 49 49 49 75 75 75 F F F
69162024-02-08T06:42:00.357Z 3 060-079 54 54 54 84 84 84 F F F
69172024-02-08T06:42:00.357Z 4 080-099 58 58 58 94 94 94 F F F
69182024-02-08T06:42:00.357Z 5 100-119 63 63 63 102 102 102 F F F
69192024-02-08T06:42:00.357Z 6 120-139 48 48 48 73 73 73 F F F
69202024-02-08T06:42:00.357Z 7 140-159 51 51 51 78 78 78 F F F
69212024-02-08T06:42:00.357Z 8 160-179 63 63 63 102 102 102 F F F
69222024-02-08T06:42:00.357Z 9 180-199 60 60 60 97 97 97 F F F
69232024-02-08T06:42:00.357Z 10 200-219 61 61 61 98 98 98 F F F
69242024-02-08T06:42:00.357Z 11 220-239 56 56 56 91 91 91 F F F
69252024-02-08T06:42:00.357Z 12 240-259 52 52 52 79 79 79 F F F
69262024-02-08T06:42:00.357Z 13 260-279 62 62 62 101 101 101 F F F
69272024-02-08T06:42:00.357Z 14 280-299 54 54 54 85 85 85 F F F
69282024-02-08T06:42:00.357Z 15 300-319 63 63 63 102 102 102 F F F
69292024-02-08T06:42:00.358Z 16 320-339 63 63 63 102 102 102 F F F
69302024-02-08T06:42:00.358Z 17 340-359 63 63 63 102 102 102 F F F
69312024-02-08T06:42:00.358Z 18 360-379 47 47 47 72 72 72 F F F
69322024-02-08T06:42:00.358Z 19 380-399 60 60 60 97 97 97 F F F
69332024-02-08T06:42:00.358Z 20 400-419 59 59 59 95 95 95 F F F
69342024-02-08T06:42:00.358Z 21 420-439 60 60 60 97 97 97 F F F
69352024-02-08T06:42:00.358Z 22 440-459 63 63 63 102 102 102 F F F
69362024-02-08T06:42:00.358Z 23 460-479 63 63 63 102 102 102 F F F
69372024-02-08T06:42:00.358Z 24 480-499 60 60 60 97 97 97 F F F
69382024-02-08T06:42:00.358Z 25 500-519 60 60 60 97 97 97 F F F
69392024-02-08T06:42:00.358Z 26 520-539 63 63 63 102 102 102 F F F
69402024-02-08T06:42:00.358Z 27 540-559 59 59 59 95 95 95 F F F
69412024-02-08T06:42:00.358Z 28 560-579 63 63 63 102 102 102 F F F
69422024-02-08T06:42:00.358Z 29 580-599 63 63 63 102 102 102 F F F
69432024-02-08T06:42:00.358ZMax gen: 63, Max flush: 102
69442024-02-08T06:42:00.358Z
69452024-02-08T06:42:00.359ZBegin loop 32
69462024-02-08T06:42:00.359ZDownstairs to restart: 1
69472024-02-08T06:42:00.371Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.612381169Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394}
69482024-02-08T06:42:00.371Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.612607432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394}
69492024-02-08T06:42:00.377Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.61263167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394}
69502024-02-08T06:42:00.377Z{"msg":"Crucible 2551964d-687a-4dfb-a89e-3a32acde5549 has session id: b6f38907-9cae-4b03-beac-58d7a0cd0724","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.612674137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69512024-02-08T06:42:00.377ZCrucible runtime is spawned
69522024-02-08T06:42:00.377Z{"msg":"Upstairs opts: Upstairs UUID: 2551964d-687a-4dfb-a89e-3a32acde5549, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.612729403Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69532024-02-08T06:42:00.377Z{"msg":"Crucible stats registered with UUID: 2551964d-687a-4dfb-a89e-3a32acde5549","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.61276741Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69542024-02-08T06:42:00.377Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.612789918Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394}
69552024-02-08T06:42:00.377Z{"msg":"2551964d-687a-4dfb-a89e-3a32acde5549 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.612873912Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69562024-02-08T06:42:00.377Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.61289922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69572024-02-08T06:42:00.377Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.612922249Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69582024-02-08T06:42:00.377Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.612948107Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69592024-02-08T06:42:00.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.612993723Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69602024-02-08T06:42:00.378Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613021831Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69612024-02-08T06:42:00.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613057318Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69622024-02-08T06:42:00.378Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613107355Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"io task","client":"2","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69632024-02-08T06:42:00.378Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613149281Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"io task","client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69642024-02-08T06:42:00.378Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613177919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"io task","client":"0","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69652024-02-08T06:42:00.378Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613202507Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"io task","client":"2","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69662024-02-08T06:42:00.378Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613244944Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"io task","client":"0","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69672024-02-08T06:42:00.378Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:59.613277282Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"io task","client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69682024-02-08T06:42:00.378Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:59.613338237Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"io task","client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69692024-02-08T06:42:00.378Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:41:59.613446869Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69702024-02-08T06:42:00.378Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613484746Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69712024-02-08T06:42:00.378Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613508354Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69722024-02-08T06:42:00.378Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613541302Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"io task","client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69732024-02-08T06:42:00.378Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613877787Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69742024-02-08T06:42:00.379Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613924183Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69752024-02-08T06:42:00.379Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613949371Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69762024-02-08T06:42:00.379Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.613989238Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69772024-02-08T06:42:00.379Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.614470852Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69782024-02-08T06:42:00.379Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.614583854Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69792024-02-08T06:42:00.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.614949236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69802024-02-08T06:42:00.379Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.615003322Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69812024-02-08T06:42:00.379Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.615051139Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69822024-02-08T06:42:00.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.615084326Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69832024-02-08T06:42:00.379Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.615102015Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69842024-02-08T06:42:00.379Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:41:59.615116324Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69852024-02-08T06:42:01.375Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.614822858Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"io task","client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69862024-02-08T06:42:01.382Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.615034692Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"io task","client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69872024-02-08T06:42:01.383Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.615785216Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69882024-02-08T06:42:01.383Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.615823913Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69892024-02-08T06:42:01.383Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.616310496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69902024-02-08T06:42:01.383Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.616841436Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69912024-02-08T06:42:01.383Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.616876094Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69922024-02-08T06:42:01.383Z{"msg":"[0]R flush_numbers[0..12]: [95, 102, 75, 84, 94, 102, 73, 78, 102, 97, 98, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.616892353Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69932024-02-08T06:42:01.383Z{"msg":"[0]R generation[0..12]: [59, 63, 49, 54, 58, 63, 48, 51, 63, 60, 61, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.61692644Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69942024-02-08T06:42:01.383Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.616944678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69952024-02-08T06:42:01.384Z{"msg":"[1]R flush_numbers[0..12]: [95, 102, 75, 84, 94, 102, 73, 78, 102, 97, 98, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.616975976Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69962024-02-08T06:42:01.384Z{"msg":"[1]R generation[0..12]: [59, 63, 49, 54, 58, 63, 48, 51, 63, 60, 61, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.616996725Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69972024-02-08T06:42:01.384Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617030822Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO
69982024-02-08T06:42:01.384Z"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
69992024-02-08T06:42:01.384Z{"msg":"[2]R flush_numbers[0..12]: [95, 102, 75, 84, 94, 102, 73, 78, 102, 97, 98, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617068099Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70002024-02-08T06:42:01.384Z{"msg":"[2]R generation[0..12]: [59, 63, 49, 54, 58, 63, 48, 51, 63, 60, 61, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617100217Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70012024-02-08T06:42:01.384Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617132584Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70022024-02-08T06:42:01.384Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617167182Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70032024-02-08T06:42:01.384Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.61718858Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70042024-02-08T06:42:01.385Z{"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617226907Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70052024-02-08T06:42:01.385Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617243306Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70062024-02-08T06:42:01.385Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617273674Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70072024-02-08T06:42:01.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617288173Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70082024-02-08T06:42:01.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617306491Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70092024-02-08T06:42:01.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617334329Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70102024-02-08T06:42:01.385Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617349718Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70112024-02-08T06:42:01.385Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617379036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70122024-02-08T06:42:01.386Z{"msg":"2551964d-687a-4dfb-a89e-3a32acde5549 is now active with session: b6f38907-9cae-4b03-beac-58d7a0cd0724","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617395135Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70132024-02-08T06:42:01.386Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617423353Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394,"session_id":"b6f38907-9cae-4b03-beac-58d7a0cd0724"}
70142024-02-08T06:42:01.386Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:00.617437692Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394}
70152024-02-08T06:42:01.386ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
70162024-02-08T06:42:01.386ZLoading write count information from file "/tmp/test_repair_verify.data"
70172024-02-08T06:42:01.386ZRun Repair workload
70182024-02-08T06:42:01.386Z01/30 Flush
70192024-02-08T06:42:01.386Z02/30 Write block 202 len 4096 data: 8
70202024-02-08T06:42:01.386Z03/30 Read block 261 len 20480
70212024-02-08T06:42:01.387Z04/30 Read block 542 len 16384
70222024-02-08T06:42:01.393Z05/30 Read block 21 len 24576
70232024-02-08T06:42:01.393Z06/30 Read block 338 len 24576
70242024-02-08T06:42:01.393Z07/30 Read block 29 len 4096
70252024-02-08T06:42:01.393Z08/30 Read block 60 len 12288
70262024-02-08T06:42:01.393Z09/30 Write block 523 len 40960 data: 6 5 6 6 6 6 7 8 7 7
70272024-02-08T06:42:01.393Z10/30 Write block 528 len 20480 data: 7 8 9 8 8
70282024-02-08T06:42:01.393Z11/30 Read block 348 len 28672
70292024-02-08T06:42:01.394Z12/30 Write block 207 len 32768 data: 6 4 5 5 7 9 9 9
70302024-02-08T06:42:01.394Z13/30 Read block 257 len 12288
70312024-02-08T06:42:01.394Z14/30 Write block 472 len 32768 data: 8 7 8 8 4 3 3 3
70322024-02-08T06:42:01.394Z15/30 Read block 137 len 20480
70332024-02-08T06:42:01.394Z16/30 Read block 377 len 36864
70342024-02-08T06:42:01.399Z17/30 Read block 212 len 8192
70352024-02-08T06:42:01.399Z18/30 Read block 260 len 24576
70362024-02-08T06:42:01.399Z19/30 Flush
70372024-02-08T06:42:01.399Z20/30 Write block 320 len 28672 data: 10 8 8 8 6 7 5
70382024-02-08T06:42:01.399Z21/30 Read block 20 len 32768
70392024-02-08T06:42:01.403Z22/30 Write block 220 len 40960 data: 9 10 10 10 12 11 10 8 10 11
70402024-02-08T06:42:01.403Z23/30 Read block 439 len 12288
70412024-02-08T06:42:01.403Z24/30 Flush
70422024-02-08T06:42:01.403Z25/30 Write block 165 len 36864 data: 8 8 8 7 8 7 7 8 8
70432024-02-08T06:42:01.403Z26/30 Read block 429 len 8192
70442024-02-08T06:42:01.403Z27/30 Write block 185 len 16384 data: 7 7 6 6
70452024-02-08T06:42:01.408Z28/30 Read block 188 len 20480
70462024-02-08T06:42:01.408Z29/30 Read block 290 len 16384
70472024-02-08T06:42:01.408Z30/30 Read block 145 len 28672
70482024-02-08T06:42:01.408Z----------------------------------------------------------------
70492024-02-08T06:42:01.408Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
70502024-02-08T06:42:01.408ZStates: Active Active Active
70512024-02-08T06:42:01.408ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
70522024-02-08T06:42:01.408Z 1 Acked 1000 Flush 0 Done Sent Done false
70532024-02-08T06:42:01.408Z 2 Acked 1001 Write 1 Done Sent Done false
70542024-02-08T06:42:01.408Z 3 Acked 1002 Read 5 Done Sent Done false
70552024-02-08T06:42:01.408Z 4 Acked 1003 Read 4 Done Sent Done false
70562024-02-08T06:42:01.408Z 5 Acked 1004 Read 6 Done Sent Done false
70572024-02-08T06:42:01.408Z 6 Acked 1005 Read 6 Done Sent Done false
70582024-02-08T06:42:01.408Z 7 Acked 1006 Read 1 Done Sent Done false
70592024-02-08T06:42:01.409Z 8 Acked 1007 Read 3 Done Sent Done false
70602024-02-08T06:42:01.409Z 9 Acked 1008 Write 10 Done Sent Done false
70612024-02-08T06:42:01.409Z 10 Acked 1009 Write 5 Done Sent Done false
70622024-02-08T06:42:01.409Z 11 Acked 1010 Read 7 Done Sent Done false
70632024-02-08T06:42:01.409Z 12 Acked 1011 Write 8 Done Sent Done false
70642024-02-08T06:42:01.409Z 13 Acked 1012 Read 3 Done Sent Done false
70652024-02-08T06:42:01.409Z 14 Acked 1013 Write 8 Done Sent Done false
70662024-02-08T06:42:01.409Z 15 Acked 1014 Read 5 Done Sent Done false
70672024-02-08T06:42:01.409Z 16 Acked 1015 Read 9 Done Sent Done false
70682024-02-08T06:42:01.409Z 17 Acked 1016 Read 2 Done Sent Done false
70692024-02-08T06:42:01.409Z 18 Acked 1017 Read 6 Done Sent Done false
70702024-02-08T06:42:01.409Z 19 Acked 1018 Flush 0 Done Sent Done false
70712024-02-08T06:42:01.409Z 20 Acked 1019 Write 7 Done Sent Done false
70722024-02-08T06:42:01.409Z 21 Acked 1020 Read 8 Done Sent Done false
70732024-02-08T06:42:01.410Z 22 Acked 1021 Write 10 Done Sent Done false
70742024-02-08T06:42:01.410Z 23 Acked 1022 Read 3 Done Sent Done false
70752024-02-08T06:42:01.410Z 24 Acked 1023 Flush 0 Done Sent Done false
70762024-02-08T06:42:01.410Z 25 Acked 1024 Write 9 Done Sent Done false
70772024-02-08T06:42:01.410Z 26 Acked 1025 Read 2 Done Sent Done false
70782024-02-08T06:42:01.410Z 27 Acked 1026 Write 4 Done Sent Done false
70792024-02-08T06:42:01.410Z 28 Acked 1027 Read 5 Done Sent Done false
70802024-02-08T06:42:01.410Z 29 Acked 1028 Read 4 Done Sent Done false
70812024-02-08T06:42:01.410Z 30 Acked 1029 Read 7 Done Sent Done false
70822024-02-08T06:42:01.410Z STATES DS:0 DS:1 DS:2 TOTAL
70832024-02-08T06:42:01.410Z New 0 0 0 0
70842024-02-08T06:42:01.410Z Sent 0 30 0 30
70852024-02-08T06:42:01.410Z Done 30 0 30 60
70862024-02-08T06:42:01.410Z Skipped 0 0 0 0
70872024-02-08T06:42:01.411Z Error 0 0 0 0
70882024-02-08T06:42:01.411ZLast Flush: 1023 0 1023
70892024-02-08T06:42:01.411ZDownstairs last five completed:
70902024-02-08T06:42:01.411ZUpstairs last five completed: 30 29 28 27 26
70912024-02-08T06:42:01.411Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:00.647518263Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1394}
70922024-02-08T06:42:01.411ZWrote out file "/tmp/test_repair_verify.data"
70932024-02-08T06:42:01.415Z
70942024-02-08T06:42:01.415ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70952024-02-08T06:42:01.431ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70962024-02-08T06:42:01.431Z 0 000-019 59 59 59 95 95 95 F F F
70972024-02-08T06:42:01.431Z 1 020-039 63 63 63 102 102 102 F F F
70982024-02-08T06:42:01.431Z 2 040-059 49 49 49 75 75 75 F F F
70992024-02-08T06:42:01.431Z 3 060-079 54 54 54 84 84 84 F F F
71002024-02-08T06:42:01.431Z 4 080-099 58 58 58 94 94 94 F F F
71012024-02-08T06:42:01.431Z 5 100-119 63 63 63 102 102 102 F F F
71022024-02-08T06:42:01.432Z 6 120-139 48 48 48 73 73 73 F F F
71032024-02-08T06:42:01.436Z 7 140-159 51 51 51 78 78 78 F F F
71042024-02-08T06:42:01.436Z 8 160-179 63 63 63 102 102 102 T F T <---
71052024-02-08T06:42:01.436Z 9 180-199 60 60 60 97 97 97 T F T <---
71062024-02-08T06:42:01.436Z 10 200-219 64 61 64 104 98 104 F F F <---
71072024-02-08T06:42:01.436Z 11 220-239 64 56 64 105 91 105 F F F <---
71082024-02-08T06:42:01.436Z 12 240-259 52 52 52 79 79 79 F F F
71092024-02-08T06:42:01.436Z 13 260-279 62 62 62 101 101 101 F F F
71102024-02-08T06:42:01.436Z 14 280-299 54 54 54 85 85 85 F F F
71112024-02-08T06:42:01.437Z 15 300-319 63 63 63 102 102 102 F F F
71122024-02-08T06:42:01.437Z 16 320-339 64 63 64 105 102 105 F F F <---
71132024-02-08T06:42:01.437Z 17 340-359 63 63 63 102 102 102 F F F
71142024-02-08T06:42:01.437Z 18 360-379 47 47 47 72 72 72 F F F
71152024-02-08T06:42:01.437Z 19 380-399 60 60 60 97 97 97 F F F
71162024-02-08T06:42:01.437Z 20 400-419 59 59 59 95 95 95 F F F
71172024-02-08T06:42:01.437Z 21 420-439 60 60 60 97 97 97 F F F
71182024-02-08T06:42:01.437Z 22 440-459 63 63 63 102 102 102 F F F
71192024-02-08T06:42:01.437Z 23 460-479 64 63 64 104 102 104 F F F <---
71202024-02-08T06:42:01.438Z 24 480-499 60 60 60 97 97 97 F F F
71212024-02-08T06:42:01.438Z 25 500-519 60 60 60 97 97 97 F F F
71222024-02-08T06:42:01.438Z 26 520-539 64 63 64 104 102 104 F F F <---
71232024-02-08T06:42:01.438Z 27 540-559 59 59 59 95 95 95 F F F
71242024-02-08T06:42:01.438Z 28 560-579 63 63 63 102 102 102 F F F
71252024-02-08T06:42:01.438Z 29 580-599 63 63 63 102 102 102 F F F
71262024-02-08T06:42:01.438ZMax gen: 64, Max flush: 105
71272024-02-08T06:42:01.438ZError: Difference in extent metadata found!
71282024-02-08T06:42:01.438ZA Difference in extent metadata is expected here
71292024-02-08T06:42:01.439ZOn loop 32
71302024-02-08T06:42:01.439Z
71312024-02-08T06:42:01.444ZVerifying data now
71322024-02-08T06:42:05.826ZNo change after verify
71332024-02-08T06:42:05.826ZLoop: 32 Downstairs dump after verify (and repair):
71342024-02-08T06:42:05.840ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
71352024-02-08T06:42:05.841Z 0 000-019 59 59 59 95 95 95 F F F
71362024-02-08T06:42:05.845Z 1 020-039 63 63 63 102 102 102 F F F
71372024-02-08T06:42:05.845Z 2 040-059 49 49 49 75 75 75 F F F
71382024-02-08T06:42:05.845Z 3 060-079 54 54 54 84 84 84 F F F
71392024-02-08T06:42:05.845Z 4 080-099 58 58 58 94 94 94 F F F
71402024-02-08T06:42:05.845Z 5 100-119 63 63 63 102 102 102 F F F
71412024-02-08T06:42:05.845Z 6 120-139 48 48 48 73 73 73 F F F
71422024-02-08T06:42:05.845Z 7 140-159 51 51 51 78 78 78 F F F
71432024-02-08T06:42:05.845Z 8 160-179 65 65 65 106 106 106 F F F
71442024-02-08T06:42:05.845Z 9 180-199 65 65 65 106 106 106 F F F
71452024-02-08T06:42:05.845Z 10 200-219 64 64 64 104 104 104 F F F
71462024-02-08T06:42:05.845Z 11 220-239 64 64 64 105 105 105 F F F
71472024-02-08T06:42:05.845Z 12 240-259 52 52 52 79 79 79 F F F
71482024-02-08T06:42:05.845Z 13 260-279 62 62 62 101 101 101 F F F
71492024-02-08T06:42:05.845Z 14 280-299 54 54 54 85 85 85 F F F
71502024-02-08T06:42:05.846Z 15 300-319 63 63 63 102 102 102 F F F
71512024-02-08T06:42:05.846Z 16 320-339 64 64 64 105 105 105 F F F
71522024-02-08T06:42:05.846Z 17 340-359 63 63 63 102 102 102 F F F
71532024-02-08T06:42:05.846Z 18 360-379 47 47 47 72 72 72 F F F
71542024-02-08T06:42:05.846Z 19 380-399 60 60 60 97 97 97 F F F
71552024-02-08T06:42:05.846Z 20 400-419 59 59 59 95 95 95 F F F
71562024-02-08T06:42:05.846Z 21 420-439 60 60 60 97 97 97 F F F
71572024-02-08T06:42:05.846Z 22 440-459 63 63 63 102 102 102 F F F
71582024-02-08T06:42:05.846Z 23 460-479 64 64 64 104 104 104 F F F
71592024-02-08T06:42:05.846Z 24 480-499 60 60 60 97 97 97 F F F
71602024-02-08T06:42:05.846Z 25 500-519 60 60 60 97 97 97 F F F
71612024-02-08T06:42:05.846Z 26 520-539 64 64 64 104 104 104 F F F
71622024-02-08T06:42:05.846Z 27 540-559 59 59 59 95 95 95 F F F
71632024-02-08T06:42:05.846Z 28 560-579 63 63 63 102 102 102 F F F
71642024-02-08T06:42:05.846Z 29 580-599 63 63 63 102 102 102 F F F
71652024-02-08T06:42:05.847ZMax gen: 65, Max flush: 106
71662024-02-08T06:42:05.847Z
71672024-02-08T06:42:05.847ZBegin loop 33
71682024-02-08T06:42:05.847ZDownstairs to restart: 1
71692024-02-08T06:42:05.861Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101037092Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402}
71702024-02-08T06:42:05.861Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101272735Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402}
71712024-02-08T06:42:05.867Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101289913Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402}
71722024-02-08T06:42:05.867Z{"msg":"Crucible ecfa9199-6c93-424e-99ac-8ae8a5346678 has session id: aa0a1b78-37cf-4360-9c29-27de3d8ca287","v":0Crucible runtime is spawned
71732024-02-08T06:42:05.867Z,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101379027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71742024-02-08T06:42:05.867Z{"msg":"Upstairs opts: Upstairs UUID: ecfa9199-6c93-424e-99ac-8ae8a5346678, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101437162Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71752024-02-08T06:42:05.867Z{"msg":"Crucible stats registered with UUID: ecfa9199-6c93-424e-99ac-8ae8a5346678","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101474649Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71762024-02-08T06:42:05.867Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101509947Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402}
71772024-02-08T06:42:05.868Z{"msg":"ecfa9199-6c93-424e-99ac-8ae8a5346678 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101564753Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71782024-02-08T06:42:05.868Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101618509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71792024-02-08T06:42:05.868Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101662195Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71802024-02-08T06:42:05.868Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101702712Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71812024-02-08T06:42:05.868Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.10172795Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71822024-02-08T06:42:05.868Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101758478Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71832024-02-08T06:42:05.868Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101793905Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71842024-02-08T06:42:05.868Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101817803Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"io task","client":"2","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71852024-02-08T06:42:05.868Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101850811Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"io task","client":"0","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71862024-02-08T06:42:05.868Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101877609Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71872024-02-08T06:42:05.868Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:05.101908467Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71882024-02-08T06:42:05.868Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:05.101937265Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71892024-02-08T06:42:05.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.101973422Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"io task","client":"0","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71902024-02-08T06:42:05.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.1019971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"io task","client":"2","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71912024-02-08T06:42:05.868Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:05.102039977Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71922024-02-08T06:42:05.869Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.102065395Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71932024-02-08T06:42:05.869Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.102094173Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71942024-02-08T06:42:05.869Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.102212074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71952024-02-08T06:42:05.869Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.102646151Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71962024-02-08T06:42:05.869Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.102685089Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71972024-02-08T06:42:05.869Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.102711127Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71982024-02-08T06:42:05.869Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.102751824Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
71992024-02-08T06:42:05.869Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.103292333Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72002024-02-08T06:42:05.869Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.10333Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72012024-02-08T06:42:05.869Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.103901167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72022024-02-08T06:42:05.869Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.103943054Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72032024-02-08T06:42:05.869Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.103958963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72042024-02-08T06:42:05.869Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.104058765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72052024-02-08T06:42:05.869Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.104086533Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72062024-02-08T06:42:05.869Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:05.104101612Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72072024-02-08T06:42:06.864Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.103178763Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72082024-02-08T06:42:06.864Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.103387608Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72092024-02-08T06:42:06.869Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.104238294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72102024-02-08T06:42:06.869Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.104274581Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72112024-02-08T06:42:06.869Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.104742906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72122024-02-08T06:42:06.869Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105277036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72132024-02-08T06:42:06.870Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105316513Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72142024-02-08T06:42:06.870Z{"msg":"[0]R flush_numbers[0..12]: [95, 102, 75, 84, 94, 102, 73, 78, 106, 106, 104, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105334481Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72152024-02-08T06:42:06.870Z{"msg":"[0]R generation[0..12]: [59, 63, 49, 54, 58, 63, 48, 51, 65, 65, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.10535652Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72162024-02-08T06:42:06.870Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105372019Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72172024-02-08T06:42:06.870Z{"msg":"[1]R flush_numbers[0..12]: [95, 102, 75, 84, 94, 102, 73, 78, 106, 106, 104, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105393157Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72182024-02-08T06:42:06.870Z{"msg":"[1]R generation[0..12]: [59, 63, 49, 54, 58, 63, 48, 51, 65, 65, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105407146Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72192024-02-08T06:42:06.870Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105421535Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72202024-02-08T06:42:06.870Z{"msg":"[2]R flush_numbers[0..12]: [95, 102, 75, 84, 94, 102, 73, 78, 106, 106, 104, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105458082Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72212024-02-08T06:42:06.870Z{Wait for a query_work_queue command to finish before sending IO
72222024-02-08T06:42:06.870Z"msg":"[2]R generation[0..12]: [59, 63, 49, 54, 58, 63, 48, 51, 65, 65, 64, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.10548647Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72232024-02-08T06:42:06.870Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105524187Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72242024-02-08T06:42:06.870Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105559595Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72252024-02-08T06:42:06.870Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105579373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72262024-02-08T06:42:06.870Z{"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.10561454Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72272024-02-08T06:42:06.870Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105635529Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72282024-02-08T06:42:06.871Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105671016Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72292024-02-08T06:42:06.871Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105706624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72302024-02-08T06:42:06.871Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105731182Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72312024-02-08T06:42:06.871Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.10575331Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72322024-02-08T06:42:06.871Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105788367Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72332024-02-08T06:42:06.871Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105811706Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
72342024-02-08T06:42:06.871Zsession_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72352024-02-08T06:42:06.871Z{"msg":"ecfa9199-6c93-424e-99ac-8ae8a5346678 is now active with session: aa0a1b78-37cf-4360-9c29-27de3d8ca287","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105843333Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72362024-02-08T06:42:06.871Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105861212Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402,"session_id":"aa0a1b78-37cf-4360-9c29-27de3d8ca287"}
72372024-02-08T06:42:06.871Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:06.105874241Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402}
72382024-02-08T06:42:06.871ZLoading write count information from file "/tmp/test_repair_verify.data"
72392024-02-08T06:42:06.871ZRun Repair workload
72402024-02-08T06:42:06.871Z01/30 Write block 254 len 32768 data: 6 6 6 5 4 6 7 9
72412024-02-08T06:42:06.875Z02/30 Read block 428 len 4096
72422024-02-08T06:42:06.875Z03/30 Write block 267 len 8192 data: 7 9
72432024-02-08T06:42:06.875Z04/30 Write block 453 len 28672 data: 6 5 7 7 7 7 7
72442024-02-08T06:42:06.875Z05/30 Read block 63 len 16384
72452024-02-08T06:42:06.875Z06/30 Write block 503 len 40960 data: 3 6 4 4 4 4 6 8 5 3
72462024-02-08T06:42:06.876Z07/30 Write block 264 len 16384 data: 5 6 6 8
72472024-02-08T06:42:06.876Z08/30 Read block 582 len 36864
72482024-02-08T06:42:06.876Z09/30 Write block 92 len 24576 data: 12 11 10 9 7 6
72492024-02-08T06:42:06.876Z10/30 Flush
72502024-02-08T06:42:06.879Z11/30 Write block 113 len 24576 data: 5 6 7 6 5 3
72512024-02-08T06:42:06.879Z12/30 Write block 322 len 8192 data: 9 9
72522024-02-08T06:42:06.879Z13/30 Read block 183 len 32768
72532024-02-08T06:42:06.882Z14/30 Flush
72542024-02-08T06:42:06.882Z15/30 Write block 227 len 36864 data: 9 11 12 9 6 6 6 6 6
72552024-02-08T06:42:06.882Z16/30 Read block 179 len 32768
72562024-02-08T06:42:06.885Z17/30 Read block 172 len 4096
72572024-02-08T06:42:06.885Z18/30 Write block 599 len 4096 data: 2
72582024-02-08T06:42:06.885Z19/30 Write block 279 len 20480 data: 3 3 3 3 4
72592024-02-08T06:42:06.885Z20/30 Write block 591 len 8192 data: 7 6
72602024-02-08T06:42:06.888Z21/30 Write block 493 len 4096 data: 4
72612024-02-08T06:42:06.889Z22/30 Write block 352 len 40960 data: 7 6 7 7 8 9 8 7 6 4
72622024-02-08T06:42:06.889Z23/30 Write block 51 len 4096 data: 5
72632024-02-08T06:42:06.889Z24/30 Write block 5 len 28672 data: 4 4 5 5 5 6 5
72642024-02-08T06:42:06.889Z25/30 Write block 585 len 24576 data: 5 5 6 6 8 8
72652024-02-08T06:42:06.892Z26/30 Write block 211 len 8192 data: 8 10
72662024-02-08T06:42:06.892Z27/30 Read block 362 len 8192
72672024-02-08T06:42:06.892Z28/30 Read block 32 len 24576
72682024-02-08T06:42:06.892Z29/30 Read block 146 len 28672
72692024-02-08T06:42:06.892Z30/30 Read block 406 len 24576
72702024-02-08T06:42:06.896Z----------------------------------------------------------------
72712024-02-08T06:42:06.896Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
72722024-02-08T06:42:06.896ZStates: Active Active Active
72732024-02-08T06:42:06.896ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
72742024-02-08T06:42:06.896Z 1 Acked 1000 Write 8 Done Sent Done false
72752024-02-08T06:42:06.897Z 2 Acked 1001 Read 1 Done Sent Done false
72762024-02-08T06:42:06.897Z 3 Acked 1002 Write 2 Done Sent Done false
72772024-02-08T06:42:06.897Z 4 Acked 1003 Write 7 Done Sent Done false
72782024-02-08T06:42:06.897Z 5 Acked 1004 Read 4 Done Sent Done false
72792024-02-08T06:42:06.897Z 6 Acked 1005 Write 10 Done Sent Done false
72802024-02-08T06:42:06.897Z 7 Acked 1006 Write 4 Done Sent Done false
72812024-02-08T06:42:06.897Z 8 Acked 1007 Read 9 Done Sent Done false
72822024-02-08T06:42:06.897Z 9 Acked 1008 Write 6 Done Sent Done false
72832024-02-08T06:42:06.897Z 10 Acked 1009 Flush 0 Done Sent Done false
72842024-02-08T06:42:06.897Z 11 Acked 1010 Write 6 Done Sent Done false
72852024-02-08T06:42:06.897Z 12 Acked 1011 Write 2 Done Sent Done false
72862024-02-08T06:42:06.897Z 13 Acked 1012 Read 8 Done Sent Done false
72872024-02-08T06:42:06.897Z 14 Acked 1013 Flush 0 Done Sent Done false
72882024-02-08T06:42:06.897Z 15 Acked 1014 Write 9 Done Sent Done false
72892024-02-08T06:42:06.897Z 16 Acked 1015 Read 8 Done Sent Done false
72902024-02-08T06:42:06.898Z 17 Acked 1016 Read 1 Done Sent Done false
72912024-02-08T06:42:06.898Z 18 Acked 1017 Write 1 Done Sent Done false
72922024-02-08T06:42:06.898Z 19 Acked 1018 Write 5 Done Sent Done false
72932024-02-08T06:42:06.898Z 20 Acked 1019 Write 2 Done Sent Done false
72942024-02-08T06:42:06.898Z 21 Acked 1020 Write 1 Done Sent Done false
72952024-02-08T06:42:06.898Z 22 Acked 1021 Write 10 Done Sent Done false
72962024-02-08T06:42:06.898Z 23 Acked 1022 Write 1 Done Sent Done false
72972024-02-08T06:42:06.898Z 24 Acked 1023 Write 7 Done Sent Done false
72982024-02-08T06:42:06.898Z 25 Acked 1024 Write 6 Done Sent Done false
72992024-02-08T06:42:06.898Z 26 Acked 1025 Write 2 Done Sent Done false
73002024-02-08T06:42:06.898Z 27 Acked 1026 Read 2 Done Sent Done false
73012024-02-08T06:42:06.898Z 28 Acked 1027 Read 6 Done Sent Done false
73022024-02-08T06:42:06.898Z 29 Acked 1028 Read 7 Done Sent Done false
73032024-02-08T06:42:06.898Z 30 Acked 1029 Read 6 Done Sent Done false
73042024-02-08T06:42:06.898Z STATES DS:0 DS:1 DS:2 TOTAL
73052024-02-08T06:42:06.899Z New 0 0 0 0
73062024-02-08T06:42:06.899Z Sent 0 30 0 30
73072024-02-08T06:42:06.899Z Done 30 0 30 60
73082024-02-08T06:42:06.899Z Skipped 0 0 0 0
73092024-02-08T06:42:06.899Z Error 0 0 0 0
73102024-02-08T06:42:06.899ZLast Flush: 1013 0 1013
73112024-02-08T06:42:06.899ZDownstairs last five completed:
73122024-02-08T06:42:06.899ZUpstairs last five completed: 30 29 28 27 26
73132024-02-08T06:42:06.899Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:06.133618148Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1402}
73142024-02-08T06:42:06.899ZWrote out file "/tmp/test_repair_verify.data"
73152024-02-08T06:42:06.903Z
73162024-02-08T06:42:06.903ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
73172024-02-08T06:42:06.919ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
73182024-02-08T06:42:06.919Z 0 000-019 59 59 59 95 95 95 T F T <---
73192024-02-08T06:42:06.920Z 1 020-039 63 63 63 102 102 102 F F F
73202024-02-08T06:42:06.920Z 2 040-059 49 49 49 75 75 75 T F T <---
73212024-02-08T06:42:06.920Z 3 060-079 54 54 54 84 84 84 F F F
73222024-02-08T06:42:06.923Z 4 080-099 66 58 66 107 94 107 F F F <---
73232024-02-08T06:42:06.923Z 5 100-119 66 63 66 108 102 108 F F F <---
73242024-02-08T06:42:06.923Z 6 120-139 48 48 48 73 73 73 F F F
73252024-02-08T06:42:06.923Z 7 140-159 51 51 51 78 78 78 F F F
73262024-02-08T06:42:06.923Z 8 160-179 65 65 65 106 106 106 F F F
73272024-02-08T06:42:06.923Z 9 180-199 65 65 65 106 106 106 F F F
73282024-02-08T06:42:06.923Z 10 200-219 64 64 64 104 104 104 T F T <---
73292024-02-08T06:42:06.924Z 11 220-239 64 64 64 105 105 105 T F T <---
73302024-02-08T06:42:06.924Z 12 240-259 66 52 66 107 79 107 F F F <---
73312024-02-08T06:42:06.924Z 13 260-279 66 62 66 107 101 107 T F T <---
73322024-02-08T06:42:06.924Z 14 280-299 54 54 54 85 85 85 T F T <---
73332024-02-08T06:42:06.924Z 15 300-319 63 63 63 102 102 102 F F F
73342024-02-08T06:42:06.924Z 16 320-339 66 64 66 108 105 108 F F F <---
73352024-02-08T06:42:06.924Z 17 340-359 63 63 63 102 102 102 T F T <---
73362024-02-08T06:42:06.924Z 18 360-379 47 47 47 72 72 72 T F T <---
73372024-02-08T06:42:06.924Z 19 380-399 60 60 60 97 97 97 F F F
73382024-02-08T06:42:06.924Z 20 400-419 59 59 59 95 95 95 F F F
73392024-02-08T06:42:06.924Z 21 420-439 60 60 60 97 97 97 F F F
73402024-02-08T06:42:06.924Z 22 440-459 66 63 66 107 102 107 F F F <---
73412024-02-08T06:42:06.924Z 23 460-479 64 64 64 104 104 104 F F F
73422024-02-08T06:42:06.924Z 24 480-499 60 60 60 97 97 97 T F T <---
73432024-02-08T06:42:06.924Z 25 500-519 66 60 66 107 97 107 F F F <---
73442024-02-08T06:42:06.925Z 26 520-539 64 64 64 104 104 104 F F F
73452024-02-08T06:42:06.925Z 27 540-559 59 59 59 95 95 95 F F F
73462024-02-08T06:42:06.925Z 28 560-579 63 63 63 102 102 102 F F F
73472024-02-08T06:42:06.925Z 29 580-599 63 63 63 102 102 102 T F T <---
73482024-02-08T06:42:06.925ZMax gen: 66, Max flush: 108
73492024-02-08T06:42:06.925ZError: Difference in extent metadata found!
73502024-02-08T06:42:06.925ZA Difference in extent metadata is expected here
73512024-02-08T06:42:06.925ZOn loop 33
73522024-02-08T06:42:06.925Z
73532024-02-08T06:42:06.929ZVerifying data now
73542024-02-08T06:42:11.676ZNo change after verify
73552024-02-08T06:42:11.677ZLoop: 33 Downstairs dump after verify (and repair):
73562024-02-08T06:42:11.692ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
73572024-02-08T06:42:11.692Z 0 000-019 67 67 67 109 109 109 F F F
73582024-02-08T06:42:11.692Z 1 020-039 63 63 63 102 102 102 F F F
73592024-02-08T06:42:11.692Z 2 040-059 67 67 67 109 109 109 F F F
73602024-02-08T06:42:11.692Z 3 060-079 54 54 54 84 84 84 F F F
73612024-02-08T06:42:11.692Z 4 080-099 66 66 66 107 107 107 F F F
73622024-02-08T06:42:11.692Z 5 100-119 66 66 66 108 108 108 F F F
73632024-02-08T06:42:11.692Z 6 120-139 48 48 48 73 73 73 F F F
73642024-02-08T06:42:11.692Z 7 140-159 51 51 51 78 78 78 F F F
73652024-02-08T06:42:11.692Z 8 160-179 65 65 65 106 106 106 F F F
73662024-02-08T06:42:11.692Z 9 180-199 65 65 65 106 106 106 F F F
73672024-02-08T06:42:11.692Z 10 200-219 67 67 67 109 109 109 F F F
73682024-02-08T06:42:11.692Z 11 220-239 67 67 67 109 109 109 F F F
73692024-02-08T06:42:11.693Z 12 240-259 66 66 66 107 107 107 F F F
73702024-02-08T06:42:11.696Z 13 260-279 67 67 67 109 109 109 F F F
73712024-02-08T06:42:11.696Z 14 280-299 67 67 67 109 109 109 F F F
73722024-02-08T06:42:11.696Z 15 300-319 63 63 63 102 102 102 F F F
73732024-02-08T06:42:11.696Z 16 320-339 66 66 66 108 108 108 F F F
73742024-02-08T06:42:11.696Z 17 340-359 67 67 67 109 109 109 F F F
73752024-02-08T06:42:11.696Z 18 360-379 67 67 67 109 109 109 F F F
73762024-02-08T06:42:11.696Z 19 380-399 60 60 60 97 97 97 F F F
73772024-02-08T06:42:11.696Z 20 400-419 59 59 59 95 95 95 F F F
73782024-02-08T06:42:11.697Z 21 420-439 60 60 60 97 97 97 F F F
73792024-02-08T06:42:11.697Z 22 440-459 66 66 66 107 107 107 F F F
73802024-02-08T06:42:11.697Z 23 460-479 64 64 64 104 104 104 F F F
73812024-02-08T06:42:11.697Z 24 480-499 67 67 67 109 109 109 F F F
73822024-02-08T06:42:11.697Z 25 500-519 66 66 66 107 107 107 F F F
73832024-02-08T06:42:11.697Z 26 520-539 64 64 64 104 104 104 F F F
73842024-02-08T06:42:11.697Z 27 540-559 59 59 59 95 95 95 F F F
73852024-02-08T06:42:11.697Z 28 560-579 63 63 63 102 102 102 F F F
73862024-02-08T06:42:11.697Z 29 580-599 67 67 67 109 109 109 F F F
73872024-02-08T06:42:11.697ZMax gen: 67, Max flush: 109
73882024-02-08T06:42:11.697Z
73892024-02-08T06:42:11.697ZBegin loop 34
73902024-02-08T06:42:11.697ZDownstairs to restart: 2
73912024-02-08T06:42:11.714Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954163202Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410}
73922024-02-08T06:42:11.714Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":Crucible runtime is spawned
73932024-02-08T06:42:11.720Z30,"time":"2024-02-08T06:42:10.954464659Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410}
73942024-02-08T06:42:11.720Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954509126Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410}
73952024-02-08T06:42:11.720Z{"msg":"Crucible 9f3f55de-3a72-4356-b16c-71eae175fa0e has session id: 62526759-3345-4359-918e-f4e428f83d53","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954529314Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
73962024-02-08T06:42:11.720Z{"msg":"Upstairs opts: Upstairs UUID: 9f3f55de-3a72-4356-b16c-71eae175fa0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954566622Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
73972024-02-08T06:42:11.720Z{"msg":"Crucible stats registered with UUID: 9f3f55de-3a72-4356-b16c-71eae175fa0e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954606829Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
73982024-02-08T06:42:11.720Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954625487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410}
73992024-02-08T06:42:11.720Z{"msg":"9f3f55de-3a72-4356-b16c-71eae175fa0e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954645746Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74002024-02-08T06:42:11.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954686423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74012024-02-08T06:42:11.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954737309Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74022024-02-08T06:42:11.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954774266Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74032024-02-08T06:42:11.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954842171Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74042024-02-08T06:42:11.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954886088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74052024-02-08T06:42:11.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954918395Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74062024-02-08T06:42:11.721Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.954949463Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"io task","client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74072024-02-08T06:42:11.721Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.9549818Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"io task","client":"0","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74082024-02-08T06:42:11.721Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.955038356Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"io task","client":"1","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74092024-02-08T06:42:11.721Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:10.955079563Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"io task","client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74102024-02-08T06:42:11.721Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:10.955131049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"io task","client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74112024-02-08T06:42:11.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.955166197Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"io task","client":"1","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74122024-02-08T06:42:11.721Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:10.955220043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74132024-02-08T06:42:11.721Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.95525956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74142024-02-08T06:42:11.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.955290327Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"io task","client":"0","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74152024-02-08T06:42:11.721Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.955321595Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74162024-02-08T06:42:11.721Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.955365162Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"io task","client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74172024-02-08T06:42:11.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.955774491Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74182024-02-08T06:42:11.721Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.955818918Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74192024-02-08T06:42:11.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.955848606Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74202024-02-08T06:42:11.721Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.95592325Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74212024-02-08T06:42:11.721Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.956399464Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74222024-02-08T06:42:11.722Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.956566761Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74232024-02-08T06:42:11.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.956948183Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74242024-02-08T06:42:11.722Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.957011758Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74252024-02-08T06:42:11.722Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.957049505Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74262024-02-08T06:42:11.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.957073163Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74272024-02-08T06:42:11.722Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.957095812Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74282024-02-08T06:42:11.722Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:10.95712062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74292024-02-08T06:42:12.830Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.955734876Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"io task","client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74302024-02-08T06:42:12.837Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.955959389Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"io task","client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74312024-02-08T06:42:12.837Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.956898099Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74322024-02-08T06:42:12.837Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.956934096Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74332024-02-08T06:42:12.837Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.957421169Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74342024-02-08T06:42:12.837Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.95794646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74352024-02-08T06:42:12.837Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.957988037Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74362024-02-08T06:42:12.837Z{"msg":"[0]R flush_numbers[0..12]: [109, 102, 109, 84, 107, 108, 73, 78, 106, 106, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958007735Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74372024-02-08T06:42:12.837Z{"msg":"[0]R generation[0..12]: [67, 63, 67, 54, 66, 66, 48, 51, 65, 65, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958034423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74382024-02-08T06:42:12.837Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958053322Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74392024-02-08T06:42:12.837Z{"msg":"[1]R flush_numbers[0..12]: [109, 102, 109, 84, 107, 108, 73, 78, 106, 106, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.95808053Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74402024-02-08T06:42:12.837Z{"msg":"[1]R generation[0..12]: [67, 63, 67, 54, 66, 66, 48, 51, 65, 65, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958099568Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74412024-02-08T06:42:12.837Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958125586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74422024-02-08T06:42:12.837Z{"msg":"[2]R flush_numbers[0..12]: [109, 102, 109, 84, 107, 108, 73, 78, 106, 106, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958144455Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74432024-02-08T06:42:12.837Z{"msg":"[2]R generation[0..12]: [67, 63, 67, 54, 66, 66, 48, 51, 65, 65, 67, 67]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
74442024-02-08T06:42:12.838Ztime":"2024-02-08T06:42:11.958170033Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74452024-02-08T06:42:12.838Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958200231Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74462024-02-08T06:42:12.838Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958225619Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74472024-02-08T06:42:12.838Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958244537Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74482024-02-08T06:42:12.838Z{"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958269036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74492024-02-08T06:42:12.838Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958296354Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74502024-02-08T06:42:12.838Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958311443Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74512024-02-08T06:42:12.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958324951Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74522024-02-08T06:42:12.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.95834497Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74532024-02-08T06:42:12.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958358419Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74542024-02-08T06:42:12.838Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958379577Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74552024-02-08T06:42:12.838Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958392486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74562024-02-08T06:42:12.838Z{"msg":"9f3f55de-3a72-4356-b16c-71eae175fa0e is now active with session: 62526759-3345-4359-918e-f4e428f83d53","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958411865Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74572024-02-08T06:42:12.839Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958424224Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410,"session_id":"62526759-3345-4359-918e-f4e428f83d53"}
74582024-02-08T06:42:12.839Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:11.958443522Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410}
74592024-02-08T06:42:12.839ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
74602024-02-08T06:42:12.839ZLoading write count information from file "/tmp/test_repair_verify.data"
74612024-02-08T06:42:12.839ZRun Repair workload
74622024-02-08T06:42:12.839Z01/30 Write block 328 len 16384 data: 3 3 3 6
74632024-02-08T06:42:12.839Z02/30 Write block 189 len 20480 data: 6 6 7 7 8
74642024-02-08T06:42:12.839Z03/30 Write block 549 len 40960 data: 8 6 7 6 7 7 6 9 7 5
74652024-02-08T06:42:12.839Z04/30 Write block 238 len 16384 data: 5 6 5 6
74662024-02-08T06:42:12.839Z05/30 Write block 272 len 8192 data: 7 7
74672024-02-08T06:42:12.840Z06/30 Write block 120 len 8192 data: 2 2
74682024-02-08T06:42:12.840Z07/30 Write block 369 len 4096 data: 6
74692024-02-08T06:42:12.840Z08/30 Read block 362 len 36864
74702024-02-08T06:42:12.840Z09/30 Read block 589 len 20480
74712024-02-08T06:42:12.840Z10/30 Read block 184 len 40960
74722024-02-08T06:42:12.840Z11/30 Read block 416 len 20480
74732024-02-08T06:42:12.840Z12/30 Read block 526 len 32768
74742024-02-08T06:42:12.840Z13/30 Flush
74752024-02-08T06:42:12.840Z14/30 Write block 300 len 24576 data: 9 9 9 8 7 7
74762024-02-08T06:42:12.840Z15/30 Write block 132 len 28672 data: 6 6 4 4 5 7 5
74772024-02-08T06:42:12.840Z16/30 Read block 186 len 32768
74782024-02-08T06:42:12.840Z17/30 Read block 346 len 40960
74792024-02-08T06:42:12.840Z18/30 Read block 349 len 16384
74802024-02-08T06:42:12.840Z19/30 Read block 78 len 40960
74812024-02-08T06:42:12.840Z20/30 Write block 513 len 16384 data: 3 3 2 3
74822024-02-08T06:42:12.840Z21/30 Write block 273 len 20480 data: 8 6 6 6 6
74832024-02-08T06:42:12.841Z22/30 Read block 15 len 12288
74842024-02-08T06:42:12.841Z23/30 Write block 328 len 36864 data: 4 4 4 7 7 8 9 7 9
74852024-02-08T06:42:12.841Z24/30 Write block 362 len 12288 data: 4 3 4
74862024-02-08T06:42:12.841Z25/30 Write block 359 len 4096 data: 8
74872024-02-08T06:42:12.841Z26/30 Write block 46 len 4096 data: 4
74882024-02-08T06:42:12.841Z27/30 Read block 454 len 28672
74892024-02-08T06:42:12.841Z28/30 Read block 470 len 8192
74902024-02-08T06:42:12.841Z29/30 Read block 69 len 12288
74912024-02-08T06:42:12.841Z30/30 Read block 13 len 8192
74922024-02-08T06:42:12.841Z----------------------------------------------------------------
74932024-02-08T06:42:12.841Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
74942024-02-08T06:42:12.841ZStates: Active Active Active
74952024-02-08T06:42:12.842ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74962024-02-08T06:42:12.842Z 1 Acked 1000 Write 4 Done Done Sent false
74972024-02-08T06:42:12.842Z 2 Acked 1001 Write 5 Done Done Sent false
74982024-02-08T06:42:12.842Z 3 Acked 1002 Write 10 Done Done Sent false
74992024-02-08T06:42:12.842Z 4 Acked 1003 Write 4 Done Done Sent false
75002024-02-08T06:42:12.842Z 5 Acked 1004 Write 2 Done Done Sent false
75012024-02-08T06:42:12.842Z 6 Acked 1005 Write 2 Done Done Sent false
75022024-02-08T06:42:12.842Z 7 Acked 1006 Write 1 Done Done Sent false
75032024-02-08T06:42:12.842Z 8 Acked 1007 Read 9 Done Done Sent false
75042024-02-08T06:42:12.842Z 9 Acked 1008 Read 5 Done Done Sent false
75052024-02-08T06:42:12.842Z 10 Acked 1009 Read 10 Done Done Sent false
75062024-02-08T06:42:12.842Z 11 Acked 1010 Read 5 Done Done Sent false
75072024-02-08T06:42:12.842Z 12 Acked 1011 Read 8 Done Done Sent false
75082024-02-08T06:42:12.842Z 13 Acked 1012 Flush 0 Done Done Sent false
75092024-02-08T06:42:12.842Z 14 Acked 1013 Write 6 Done Done Sent false
75102024-02-08T06:42:12.842Z 15 Acked 1014 Write 7 Done Done Sent false
75112024-02-08T06:42:12.842Z 16 Acked 1015 Read 8 Done Done Sent false
75122024-02-08T06:42:12.843Z 17 Acked 1016 Read 10 Done Done Sent false
75132024-02-08T06:42:12.843Z 18 Acked 1017 Read 4 Done Done Sent false
75142024-02-08T06:42:12.843Z 19 Acked 1018 Read 10 Done Done Sent false
75152024-02-08T06:42:12.843Z 20 Acked 1019 Write 4 Done Done Sent false
75162024-02-08T06:42:12.843Z 21 Acked 1020 Write 5 Done Done Sent false
75172024-02-08T06:42:12.843Z 22 Acked 1021 Read 3 Done Done Sent false
75182024-02-08T06:42:12.843Z 23 Acked 1022 Write 9 Done Done Sent false
75192024-02-08T06:42:12.843Z 24 Acked 1023 Write 3 Done Done Sent false
75202024-02-08T06:42:12.843Z 25 Acked 1024 Write 1 Done Done Sent false
75212024-02-08T06:42:12.843Z 26 Acked 1025 Write 1 Done Done Sent false
75222024-02-08T06:42:12.843Z 27 Acked 1026 Read 7 Done Done Sent false
75232024-02-08T06:42:12.843Z 28 Acked 1027 Read 2 Done Done Sent false
75242024-02-08T06:42:12.843Z 29 Acked 1028 Read 3 Done Done Sent false
75252024-02-08T06:42:12.843Z 30 Acked 1029 Read 2 Done Done Sent false
75262024-02-08T06:42:12.843Z STATES DS:0 DS:1 DS:2 TOTAL
75272024-02-08T06:42:12.843Z New 0 0 0 0
75282024-02-08T06:42:12.843Z Sent 0 0 30 30
75292024-02-08T06:42:12.843Z Done 30 30 0 60
75302024-02-08T06:42:12.848Z Skipped 0 0 0 0
75312024-02-08T06:42:12.848Z Error 0 0 0 0
75322024-02-08T06:42:12.848ZLast Flush: 1012 1012 0
75332024-02-08T06:42:12.848ZDownstairs last five completed:
75342024-02-08T06:42:12.848ZUpstairs last five completed: 30 29 28 27 26
75352024-02-08T06:42:12.848Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:11.984371856Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1410}
75362024-02-08T06:42:12.848ZWrote out file "/tmp/test_repair_verify.data"
75372024-02-08T06:42:12.848Z
75382024-02-08T06:42:12.848ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
75392024-02-08T06:42:12.848ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75402024-02-08T06:42:12.848Z 0 000-019 67 67 67 109 109 109 F F F
75412024-02-08T06:42:12.848Z 1 020-039 63 63 63 102 102 102 F F F
75422024-02-08T06:42:12.849Z 2 040-059 67 67 67 109 109 109 T T F <---
75432024-02-08T06:42:12.849Z 3 060-079 54 54 54 84 84 84 F F F
75442024-02-08T06:42:12.849Z 4 080-099 66 66 66 107 107 107 F F F
75452024-02-08T06:42:12.849Z 5 100-119 66 66 66 108 108 108 F F F
75462024-02-08T06:42:12.849Z 6 120-139 68 68 48 110 110 73 T T F <---
75472024-02-08T06:42:12.849Z 7 140-159 51 51 51 78 78 78 F F F
75482024-02-08T06:42:12.849Z 8 160-179 65 65 65 106 106 106 F F F
75492024-02-08T06:42:12.849Z 9 180-199 68 68 65 110 110 106 F F F <---
75502024-02-08T06:42:12.849Z 10 200-219 67 67 67 109 109 109 F F F
75512024-02-08T06:42:12.849Z 11 220-239 68 68 67 110 110 109 F F F <---
75522024-02-08T06:42:12.849Z 12 240-259 68 68 66 110 110 107 F F F <---
75532024-02-08T06:42:12.849Z 13 260-279 68 68 67 110 110 109 T T F <---
75542024-02-08T06:42:12.849Z 14 280-299 67 67 67 109 109 109 F F F
75552024-02-08T06:42:12.849Z 15 300-319 63 63 63 102 102 102 T T F <---
75562024-02-08T06:42:12.849Z 16 320-339 68 68 66 110 110 108 T T F <---
75572024-02-08T06:42:12.850Z 17 340-359 67 67 67 109 109 109 T T F <---
75582024-02-08T06:42:12.850Z 18 360-379 68 68 67 110 110 109 T T F <---
75592024-02-08T06:42:12.850Z 19 380-399 60 60 60 97 97 97 F F F
75602024-02-08T06:42:12.850Z 20 400-419 59 59 59 95 95 95 F F F
75612024-02-08T06:42:12.850Z 21 420-439 60 60 60 97 97 97 F F F
75622024-02-08T06:42:12.850Z 22 440-459 66 66 66 107 107 107 F F F
75632024-02-08T06:42:12.850Z 23 460-479 64 64 64 104 104 104 F F F
75642024-02-08T06:42:12.850Z 24 480-499 67 67 67 109 109 109 F F F
75652024-02-08T06:42:12.850Z 25 500-519 66 66 66 107 107 107 T T F <---
75662024-02-08T06:42:12.850Z 26 520-539 64 64 64 104 104 104 F F F
75672024-02-08T06:42:12.850Z 27 540-559 68 68 59 110 110 95 F F F <---
75682024-02-08T06:42:12.850Z 28 560-579 63 63 63 102 102 102 F F F
75692024-02-08T06:42:12.850Z 29 580-599 67 67 67 109 109 109 F F F
75702024-02-08T06:42:12.850ZMax gen: 68, Max flush: 110
75712024-02-08T06:42:12.850ZError: Difference in extent metadata found!
75722024-02-08T06:42:12.850ZA Difference in extent metadata is expected here
75732024-02-08T06:42:12.851ZOn loop 34
75742024-02-08T06:42:12.851Z
75752024-02-08T06:42:12.851ZVerifying data now
75762024-02-08T06:42:17.368ZNo change after verify
75772024-02-08T06:42:17.368ZLoop: 34 Downstairs dump after verify (and repair):
75782024-02-08T06:42:17.383ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75792024-02-08T06:42:17.383Z 0 000-019 67 67 67 109 109 109 F F F
75802024-02-08T06:42:17.383Z 1 020-039 63 63 63 102 102 102 F F F
75812024-02-08T06:42:17.383Z 2 040-059 69 69 69 111 111 111 F F F
75822024-02-08T06:42:17.383Z 3 060-079 54 54 54 84 84 84 F F F
75832024-02-08T06:42:17.383Z 4 080-099 66 66 66 107 107 107 F F F
75842024-02-08T06:42:17.387Z 5 100-119 66 66 66 108 108 108 F F F
75852024-02-08T06:42:17.387Z 6 120-139 69 69 69 111 111 111 F F F
75862024-02-08T06:42:17.387Z 7 140-159 51 51 51 78 78 78 F F F
75872024-02-08T06:42:17.387Z 8 160-179 65 65 65 106 106 106 F F F
75882024-02-08T06:42:17.387Z 9 180-199 68 68 68 110 110 110 F F F
75892024-02-08T06:42:17.387Z 10 200-219 67 67 67 109 109 109 F F F
75902024-02-08T06:42:17.387Z 11 220-239 68 68 68 110 110 110 F F F
75912024-02-08T06:42:17.387Z 12 240-259 68 68 68 110 110 110 F F F
75922024-02-08T06:42:17.387Z 13 260-279 69 69 69 111 111 111 F F F
75932024-02-08T06:42:17.387Z 14 280-299 67 67 67 109 109 109 F F F
75942024-02-08T06:42:17.387Z 15 300-319 69 69 69 111 111 111 F F F
75952024-02-08T06:42:17.387Z 16 320-339 69 69 69 111 111 111 F F F
75962024-02-08T06:42:17.388Z 17 340-359 69 69 69 111 111 111 F F F
75972024-02-08T06:42:17.388Z 18 360-379 69 69 69 111 111 111 F F F
75982024-02-08T06:42:17.388Z 19 380-399 60 60 60 97 97 97 F F F
75992024-02-08T06:42:17.388Z 20 400-419 59 59 59 95 95 95 F F F
76002024-02-08T06:42:17.388Z 21 420-439 60 60 60 97 97 97 F F F
76012024-02-08T06:42:17.388Z 22 440-459 66 66 66 107 107 107 F F F
76022024-02-08T06:42:17.388Z 23 460-479 64 64 64 104 104 104 F F F
76032024-02-08T06:42:17.388Z 24 480-499 67 67 67 109 109 109 F F F
76042024-02-08T06:42:17.388Z 25 500-519 69 69 69 111 111 111 F F F
76052024-02-08T06:42:17.388Z 26 520-539 64 64 64 104 104 104 F F F
76062024-02-08T06:42:17.388Z 27 540-559 68 68 68 110 110 110 F F F
76072024-02-08T06:42:17.388Z 28 560-579 63 63 63 102 102 102 F F F
76082024-02-08T06:42:17.389Z 29 580-599 67 67 67 109 109 109 F F F
76092024-02-08T06:42:17.389ZMax gen: 69, Max flush: 111
76102024-02-08T06:42:17.389Z
76112024-02-08T06:42:17.389ZBegin loop 35
76122024-02-08T06:42:17.389ZDownstairs to restart: 0
76132024-02-08T06:42:17.405Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.643685064Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418}
76142024-02-08T06:42:17.405Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.643990411Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418}
76152024-02-08T06:42:17.405Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644022398Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418}
76162024-02-08T06:42:17.411Z{"msg":"Crucible fb359ef9-3cee-4b42-8635-34ee5f0675dd has session id: adc11438-cb47-46ab-a602-291f7017cc9a","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644041137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76172024-02-08T06:42:17.411Z{"msg":"Upstairs opts: Upstairs UUID: fb359ef9-3cee-4b42-8635-34ee5f0675dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644074055Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76182024-02-08T06:42:17.411ZCrucible runtime is spawned
76192024-02-08T06:42:17.411Z{"msg":"Crucible stats registered with UUID: fb359ef9-3cee-4b42-8635-34ee5f0675dd","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644119471Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76202024-02-08T06:42:17.411Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644189656Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418}
76212024-02-08T06:42:17.411Z{"msg":"fb359ef9-3cee-4b42-8635-34ee5f0675dd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644277119Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76222024-02-08T06:42:17.411Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644302877Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76232024-02-08T06:42:17.411Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644324136Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76242024-02-08T06:42:17.411Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644357603Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76252024-02-08T06:42:17.411Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644373802Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76262024-02-08T06:42:17.411Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.64439728Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76272024-02-08T06:42:17.411Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644442227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76282024-02-08T06:42:17.411Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644480334Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"io task","client":"2","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76292024-02-08T06:42:17.411Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644506752Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"io task","client":"1","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76302024-02-08T06:42:17.412Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644543329Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"io task","client":"0","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76312024-02-08T06:42:17.412Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644579757Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"io task","client":"2","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76322024-02-08T06:42:17.412Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644604775Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"io task","client":"1","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76332024-02-08T06:42:17.412Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.644748914Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"io task","client":"0","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76342024-02-08T06:42:17.412Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.645376237Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76352024-02-08T06:42:17.412Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.645425223Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76362024-02-08T06:42:17.412Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.645449101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76372024-02-08T06:42:17.412Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.64547075Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76382024-02-08T06:42:17.412Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.645927046Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76392024-02-08T06:42:17.412Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.646087874Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76402024-02-08T06:42:17.412Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.646117831Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76412024-02-08T06:42:17.412Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.64613491Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76422024-02-08T06:42:17.412Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.646487744Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76432024-02-08T06:42:17.412Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.64652921Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76442024-02-08T06:42:17.413Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.646548269Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76452024-02-08T06:42:17.413Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.646738945Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76462024-02-08T06:42:17.413Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.646767032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76472024-02-08T06:42:17.413Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.646782061Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76482024-02-08T06:42:17.413Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.646816609Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76492024-02-08T06:42:17.413Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647286633Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76502024-02-08T06:42:17.413Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.64733126Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76512024-02-08T06:42:17.413Z{"msg":"[0]R flush_numbers[0..12]: [109, 102, 111, 84, 107, 108, 111, 78, 106, 110, 109, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647350399Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76522024-02-08T06:42:17.413Z{"msg":"[0]R generation[0..12]: [67, 63, 69, 54, 66, 66, 69, 51, 65, 68, 67, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647366038Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76532024-02-08T06:42:17.413Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647381466Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76542024-02-08T06:42:17.413Z{"msg":"[1]R flush_numbers[0..12]: [109, 102, 111, 84, 107, 108, 111, 78, 106, 110, 109, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647395945Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76552024-02-08T06:42:17.413Z{"msg":"[1]R generation[0..12]: [67, 63, 69, 54, 66, 66, 69, 51, 65, 68, 67, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647418114Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76562024-02-08T06:42:17.413Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647433282Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76572024-02-08T06:42:17.413Z{"msg":"[2]R flush_numbers[0..12]: [109, 102, 111, 84, 107, 108, 111, 78, 106, 110, 109, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647448401Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76582024-02-08T06:42:17.413Z{"msg":"[2]R generation[0..12]: [67, 63, 69, 54, 66, 66, 69, 51, 65, 68, 67, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.64746213Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76592024-02-08T06:42:17.413Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647484309Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
76602024-02-08T06:42:17.414Zadc11438-cb47-46ab-a602-291f7017cc9a"}
76612024-02-08T06:42:17.414Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647501027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76622024-02-08T06:42:17.414Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647515646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76632024-02-08T06:42:17.414Z{"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647529025Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76642024-02-08T06:42:17.414Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647542664Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76652024-02-08T06:42:17.414Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647556543Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76662024-02-08T06:42:17.418Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647569922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76672024-02-08T06:42:17.418Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.64759197Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76682024-02-08T06:42:17.418Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.64760572Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76692024-02-08T06:42:17.418Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647619088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76702024-02-08T06:42:17.418Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647652826Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76712024-02-08T06:42:17.418Z{"msg":"fb359ef9-3cee-4b42-8635-34ee5f0675dd is now active with session: adc11438-cb47-46ab-a602-291f7017cc9a","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647667005Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76722024-02-08T06:42:17.418Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647679164Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418,"session_id":"adc11438-cb47-46ab-a602-291f7017cc9a"}
76732024-02-08T06:42:17.418Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:16.647692953Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418}
76742024-02-08T06:42:17.418ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
76752024-02-08T06:42:17.418ZLoading write count information from file "/tmp/test_repair_verify.data"
76762024-02-08T06:42:17.418ZRun Repair workload
76772024-02-08T06:42:17.418Z01/30 Write block 492 len 40960 data: 5 5 5 5 4 4 4 5 5 5
76782024-02-08T06:42:17.418Z02/30 Read block 416 len 20480
76792024-02-08T06:42:17.419Z03/30 Read block 387 len 20480
76802024-02-08T06:42:17.419Z04/30 Write block 113 len 16384 data: 6 7 8 7
76812024-02-08T06:42:17.419Z05/30 Read block 407 len 4096
76822024-02-08T06:42:17.419Z06/30 Write block 489 len 12288 data: 8 6 6
76832024-02-08T06:42:17.419Z07/30 Read block 386 len 24576
76842024-02-08T06:42:17.419Z08/30 Read block 284 len 20480
76852024-02-08T06:42:17.419Z09/30 Write block 448 len 40960 data: 7 8 7 6 6 7 6 8 8 8
76862024-02-08T06:42:17.419Z10/30 Flush
76872024-02-08T06:42:17.422Z11/30 Flush
76882024-02-08T06:42:17.422Z12/30 Write block 335 len 32768 data: 8 10 9 10 10 9 8 7
76892024-02-08T06:42:17.422Z13/30 Write block 212 len 20480 data: 11 10 10 6 6
76902024-02-08T06:42:17.425Z14/30 Write block 546 len 28672 data: 4 6 6 9 7 8 7
76912024-02-08T06:42:17.425Z15/30 Read block 474 len 12288
76922024-02-08T06:42:17.425Z16/30 Write block 207 len 20480 data: 7 5 6 6 9
76932024-02-08T06:42:17.426Z17/30 Read block 341 len 32768
76942024-02-08T06:42:17.428Z18/30 Write block 163 len 20480 data: 7 8 9 9 9
76952024-02-08T06:42:17.429Z19/30 Read block 172 len 24576
76962024-02-08T06:42:17.429Z20/30 Write block 29 len 32768 data: 5 6 5 5 7 6 6 5
76972024-02-08T06:42:17.432Z21/30 Write block 162 len 36864 data: 7 8 9 10 10 10 8 9 8
76982024-02-08T06:42:17.432Z22/30 Read block 353 len 36864
76992024-02-08T06:42:17.432Z23/30 Read block 162 len 36864
77002024-02-08T06:42:17.432Z24/30 Write block 464 len 20480 data: 4 3 3 5 5
77012024-02-08T06:42:17.435Z25/30 Flush
77022024-02-08T06:42:17.438Z26/30 Write block 576 len 28672 data: 5 5 5 8 7 6 7
77032024-02-08T06:42:17.438Z27/30 Write block 96 len 32768 data: 8 7 5 6 7 7 5 7
77042024-02-08T06:42:17.438Z28/30 Read block 116 len 24576
77052024-02-08T06:42:17.438Z29/30 Write block 6 len 12288 data: 5 6 6
77062024-02-08T06:42:17.442Z30/30 Write block 516 len 16384 data: 4 4 6 8
77072024-02-08T06:42:17.442Z----------------------------------------------------------------
77082024-02-08T06:42:17.442Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
77092024-02-08T06:42:17.442ZStates: Active Active Active
77102024-02-08T06:42:17.442ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
77112024-02-08T06:42:17.442Z 1 Acked 1000 Write 10 Sent Done Done false
77122024-02-08T06:42:17.442Z 2 Acked 1001 Read 5 Sent Done Done false
77132024-02-08T06:42:17.442Z 3 Acked 1002 Read 5 Sent Done Done false
77142024-02-08T06:42:17.442Z 4 Acked 1003 Write 4 Sent Done Done false
77152024-02-08T06:42:17.442Z 5 Acked 1004 Read 1 Sent Done Done false
77162024-02-08T06:42:17.443Z 6 Acked 1005 Write 3 Sent Done Done false
77172024-02-08T06:42:17.443Z 7 Acked 1006 Read 6 Sent Done Done false
77182024-02-08T06:42:17.443Z 8 Acked 1007 Read 5 Sent Done Done false
77192024-02-08T06:42:17.443Z 9 Acked 1008 Write 10 Sent Done Done false
77202024-02-08T06:42:17.443Z 10 Acked 1009 Flush 0 Sent Done Done false
77212024-02-08T06:42:17.443Z 11 Acked 1010 Flush 0 Sent Done Done false
77222024-02-08T06:42:17.443Z 12 Acked 1011 Write 8 Sent Done Done false
77232024-02-08T06:42:17.443Z 13 Acked 1012 Write 5 Sent Done Done false
77242024-02-08T06:42:17.443Z 14 Acked 1013 Write 7 Sent Done Done false
77252024-02-08T06:42:17.443Z 15 Acked 1014 Read 3 Sent Done Done false
77262024-02-08T06:42:17.443Z 16 Acked 1015 Write 5 Sent Done Done false
77272024-02-08T06:42:17.443Z 17 Acked 1016 Read 8 Sent Done Done false
77282024-02-08T06:42:17.443Z 18 Acked 1017 Write 5 Sent Done Done false
77292024-02-08T06:42:17.443Z 19 Acked 1018 Read 6 Sent Done Done false
77302024-02-08T06:42:17.443Z 20 Acked 1019 Write 8 Sent Done Done false
77312024-02-08T06:42:17.444Z 21 Acked 1020 Write 9 Sent Done Done false
77322024-02-08T06:42:17.444Z 22 Acked 1021 Read 9 Sent Done Done false
77332024-02-08T06:42:17.444Z 23 Acked 1022 Read 9 Sent Done Done false
77342024-02-08T06:42:17.444Z 24 Acked 1023 Write 5 Sent Done Done false
77352024-02-08T06:42:17.444Z 25 Acked 1024 Flush 0 Sent Done Done false
77362024-02-08T06:42:17.444Z 26 Acked 1025 Write 7 Sent Done Done false
77372024-02-08T06:42:17.444Z 27 Acked 1026 Write 8 Sent Done Done false
77382024-02-08T06:42:17.444Z 28 Acked 1027 Read 6 Sent Done Done false
77392024-02-08T06:42:17.444Z 29 Acked 1028 Write 3 Sent Done Sent false
77402024-02-08T06:42:17.444Z 30 Acked 1029 Write 4 Sent Sent Sent false
77412024-02-08T06:42:17.444Z STATES DS:0 DS:1 DS:2 TOTAL
77422024-02-08T06:42:17.444Z New 0 0 0 0
77432024-02-08T06:42:17.444Z Sent 30 1 2 33
77442024-02-08T06:42:17.444Z Done 0 29 28 57
77452024-02-08T06:42:17.444Z Skipped 0 0 0 0
77462024-02-08T06:42:17.444Z Error 0 0 0 0
77472024-02-08T06:42:17.445ZLast Flush: 0 1024 1024
77482024-02-08T06:42:17.445ZDownstairs last five completed:
77492024-02-08T06:42:17.445ZUpstairs last five completed: 30 29 28 27 26
77502024-02-08T06:42:17.445Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:16.678799027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1418}
77512024-02-08T06:42:17.445ZWrote out file "/tmp/test_repair_verify.data"
77522024-02-08T06:42:17.449Z
77532024-02-08T06:42:17.449ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
77542024-02-08T06:42:17.466ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77552024-02-08T06:42:17.466Z 0 000-019 67 67 67 109 109 109 F T T <---
77562024-02-08T06:42:17.466Z 1 020-039 63 70 70 102 114 114 F F F <---
77572024-02-08T06:42:17.466Z 2 040-059 69 69 69 111 111 111 F F F
77582024-02-08T06:42:17.466Z 3 060-079 54 54 54 84 84 84 F F F
77592024-02-08T06:42:17.466Z 4 080-099 66 66 66 107 107 107 F T T <---
77602024-02-08T06:42:17.466Z 5 100-119 66 70 70 108 112 112 F T T <---
77612024-02-08T06:42:17.466Z 6 120-139 69 69 69 111 111 111 F F F
77622024-02-08T06:42:17.466Z 7 140-159 51 51 51 78 78 78 F F F
77632024-02-08T06:42:17.466Z 8 160-179 65 70 70 106 114 114 F F F <---
77642024-02-08T06:42:17.466Z 9 180-199 68 68 68 110 110 110 F F F
77652024-02-08T06:42:17.467Z 10 200-219 67 70 70 109 114 114 F F F <---
77662024-02-08T06:42:17.470Z 11 220-239 68 68 68 110 110 110 F F F
77672024-02-08T06:42:17.470Z 12 240-259 68 68 68 110 110 110 F F F
77682024-02-08T06:42:17.470Z 13 260-279 69 69 69 111 111 111 F F F
77692024-02-08T06:42:17.470Z 14 280-299 67 67 67 109 109 109 F F F
77702024-02-08T06:42:17.470Z 15 300-319 69 69 69 111 111 111 F F F
77712024-02-08T06:42:17.470Z 16 320-339 69 70 70 111 114 114 F F F <---
77722024-02-08T06:42:17.470Z 17 340-359 69 70 70 111 114 114 F F F <---
77732024-02-08T06:42:17.470Z 18 360-379 69 69 69 111 111 111 F F F
77742024-02-08T06:42:17.470Z 19 380-399 60 60 60 97 97 97 F F F
77752024-02-08T06:42:17.471Z 20 400-419 59 59 59 95 95 95 F F F
77762024-02-08T06:42:17.471Z 21 420-439 60 60 60 97 97 97 F F F
77772024-02-08T06:42:17.471Z 22 440-459 66 70 70 107 112 112 F F F <---
77782024-02-08T06:42:17.471Z 23 460-479 64 70 70 104 114 114 F F F <---
77792024-02-08T06:42:17.471Z 24 480-499 67 70 70 109 112 112 F F F <---
77802024-02-08T06:42:17.471Z 25 500-519 69 70 70 111 112 112 F T T <---
77812024-02-08T06:42:17.471Z 26 520-539 64 64 64 104 104 104 F F F
77822024-02-08T06:42:17.471Z 27 540-559 68 70 70 110 114 114 F F F <---
77832024-02-08T06:42:17.471Z 28 560-579 63 63 63 102 102 102 F T T <---
77842024-02-08T06:42:17.471Z 29 580-599 67 67 67 109 109 109 F T T <---
77852024-02-08T06:42:17.471ZMax gen: 70, Max flush: 114
77862024-02-08T06:42:17.471ZError: Difference in extent metadata found!
77872024-02-08T06:42:17.471ZA Difference in extent metadata is expected here
77882024-02-08T06:42:17.471ZOn loop 35
77892024-02-08T06:42:17.471Z
77902024-02-08T06:42:17.472ZVerifying data now
77912024-02-08T06:42:22.184ZNo change after verify
77922024-02-08T06:42:22.184ZLoop: 35 Downstairs dump after verify (and repair):
77932024-02-08T06:42:22.199ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77942024-02-08T06:42:22.199Z 0 000-019 71 71 71 115 115 115 F F F
77952024-02-08T06:42:22.199Z 1 020-039 70 70 70 114 114 114 F F F
77962024-02-08T06:42:22.199Z 2 040-059 69 69 69 111 111 111 F F F
77972024-02-08T06:42:22.199Z 3 060-079 54 54 54 84 84 84 F F F
77982024-02-08T06:42:22.199Z 4 080-099 71 71 71 115 115 115 F F F
77992024-02-08T06:42:22.202Z 5 100-119 71 71 71 115 115 115 F F F
78002024-02-08T06:42:22.203Z 6 120-139 69 69 69 111 111 111 F F F
78012024-02-08T06:42:22.203Z 7 140-159 51 51 51 78 78 78 F F F
78022024-02-08T06:42:22.203Z 8 160-179 70 70 70 114 114 114 F F F
78032024-02-08T06:42:22.203Z 9 180-199 68 68 68 110 110 110 F F F
78042024-02-08T06:42:22.203Z 10 200-219 70 70 70 114 114 114 F F F
78052024-02-08T06:42:22.203Z 11 220-239 68 68 68 110 110 110 F F F
78062024-02-08T06:42:22.203Z 12 240-259 68 68 68 110 110 110 F F F
78072024-02-08T06:42:22.203Z 13 260-279 69 69 69 111 111 111 F F F
78082024-02-08T06:42:22.203Z 14 280-299 67 67 67 109 109 109 F F F
78092024-02-08T06:42:22.203Z 15 300-319 69 69 69 111 111 111 F F F
78102024-02-08T06:42:22.203Z 16 320-339 70 70 70 114 114 114 F F F
78112024-02-08T06:42:22.203Z 17 340-359 70 70 70 114 114 114 F F F
78122024-02-08T06:42:22.203Z 18 360-379 69 69 69 111 111 111 F F F
78132024-02-08T06:42:22.203Z 19 380-399 60 60 60 97 97 97 F F F
78142024-02-08T06:42:22.204Z 20 400-419 59 59 59 95 95 95 F F F
78152024-02-08T06:42:22.204Z 21 420-439 60 60 60 97 97 97 F F F
78162024-02-08T06:42:22.204Z 22 440-459 70 70 70 112 112 112 F F F
78172024-02-08T06:42:22.204Z 23 460-479 70 70 70 114 114 114 F F F
78182024-02-08T06:42:22.204Z 24 480-499 70 70 70 112 112 112 F F F
78192024-02-08T06:42:22.204Z 25 500-519 71 71 71 115 115 115 F F F
78202024-02-08T06:42:22.204Z 26 520-539 64 64 64 104 104 104 F F F
78212024-02-08T06:42:22.204Z 27 540-559 70 70 70 114 114 114 F F F
78222024-02-08T06:42:22.204Z 28 560-579 71 71 71 115 115 115 F F F
78232024-02-08T06:42:22.204Z 29 580-599 71 71 71 115 115 115 F F F
78242024-02-08T06:42:22.204ZMax gen: 71, Max flush: 115
78252024-02-08T06:42:22.204Z
78262024-02-08T06:42:22.204ZBegin loop 36
78272024-02-08T06:42:22.204ZDownstairs to restart: 0
78282024-02-08T06:42:22.219Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458393856Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426}
78292024-02-08T06:42:22.220Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458616739Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426}
78302024-02-08T06:42:22.220Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0Crucible runtime is spawned
78312024-02-08T06:42:22.225Z,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458665016Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426}
78322024-02-08T06:42:22.225Z{"msg":"Crucible 1a176afa-e5d1-4b43-99fa-b82f22a02173 has session id: 97df21ca-d251-4713-bdca-5345fbc22e78","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458701433Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78332024-02-08T06:42:22.225Z{"msg":"Upstairs opts: Upstairs UUID: 1a176afa-e5d1-4b43-99fa-b82f22a02173, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.45873293Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78342024-02-08T06:42:22.225Z{"msg":"Crucible stats registered with UUID: 1a176afa-e5d1-4b43-99fa-b82f22a02173","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458777697Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78352024-02-08T06:42:22.225Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458794416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426}
78362024-02-08T06:42:22.225Z{"msg":"1a176afa-e5d1-4b43-99fa-b82f22a02173 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458861151Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78372024-02-08T06:42:22.225Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458886929Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78382024-02-08T06:42:22.225Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458907757Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78392024-02-08T06:42:22.225Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458930765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78402024-02-08T06:42:22.225Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458945294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78412024-02-08T06:42:22.225Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.458961973Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78422024-02-08T06:42:22.226Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.45899708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78432024-02-08T06:42:22.226Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.459013829Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"io task","client":"2","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78442024-02-08T06:42:22.226Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.459068405Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"io task","client":"1","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78452024-02-08T06:42:22.226Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.45913312Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"io task","client":"0","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78462024-02-08T06:42:22.226Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.459223503Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"io task","client":"2","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78472024-02-08T06:42:22.226Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.459415159Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"io task","client":"1","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78482024-02-08T06:42:22.226Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.459472595Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"io task","client":"0","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78492024-02-08T06:42:22.226Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.459981117Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78502024-02-08T06:42:22.226Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.460030903Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78512024-02-08T06:42:22.226Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.460049201Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78522024-02-08T06:42:22.226Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.460115707Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78532024-02-08T06:42:22.226Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.460661866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78542024-02-08T06:42:22.226Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.460700772Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78552024-02-08T06:42:22.226Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.461039027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78562024-02-08T06:42:22.226Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.461087423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78572024-02-08T06:42:22.226Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.461188146Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78582024-02-08T06:42:22.227Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.461227143Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78592024-02-08T06:42:22.227Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.461242842Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78602024-02-08T06:42:22.227Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.461319956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78612024-02-08T06:42:22.227Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.461354603Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78622024-02-08T06:42:22.227Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.461369282Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78632024-02-08T06:42:22.227Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.461556178Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78642024-02-08T06:42:22.232Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462023763Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78652024-02-08T06:42:22.232Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462056901Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78662024-02-08T06:42:22.232Z{"msg":"[0]R flush_numbers[0..12]: [115, 114, 111, 84, 115, 115, 111, 78, 114, 110, 114, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.46207262Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78672024-02-08T06:42:22.232Z{"msg":"[0]R generation[0..12]: [71, 70, 69, 54, 71, 71, 69, 51, 70, 68, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462146904Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78682024-02-08T06:42:22.232Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462173242Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78692024-02-08T06:42:22.232Z{"msg":"[1]R flush_numbers[0..12]: [115, 114, 111, 84, 115, 115, 111, 78, 114, 110, 114, 110]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
78702024-02-08T06:42:22.232Z,"time":"2024-02-08T06:42:21.462190871Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78712024-02-08T06:42:22.232Z{"msg":"[1]R generation[0..12]: [71, 70, 69, 54, 71, 71, 69, 51, 70, 68, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.46220567Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78722024-02-08T06:42:22.232Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462227478Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78732024-02-08T06:42:22.232Z{"msg":"[2]R flush_numbers[0..12]: [115, 114, 111, 84, 115, 115, 111, 78, 114, 110, 114, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462242247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78742024-02-08T06:42:22.232Z{"msg":"[2]R generation[0..12]: [71, 70, 69, 54, 71, 71, 69, 51, 70, 68, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462262435Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78752024-02-08T06:42:22.232Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462276734Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78762024-02-08T06:42:22.233Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462296473Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78772024-02-08T06:42:22.233Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462310292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78782024-02-08T06:42:22.233Z{"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.46233003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78792024-02-08T06:42:22.233Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462343239Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78802024-02-08T06:42:22.233Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462363198Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78812024-02-08T06:42:22.233Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462375587Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78822024-02-08T06:42:22.233Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462395305Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78832024-02-08T06:42:22.233Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462408924Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78842024-02-08T06:42:22.233Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462428643Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78852024-02-08T06:42:22.233Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462441242Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78862024-02-08T06:42:22.233Z{"msg":"1a176afa-e5d1-4b43-99fa-b82f22a02173 is now active with session: 97df21ca-d251-4713-bdca-5345fbc22e78","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462454201Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78872024-02-08T06:42:22.233Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.46247287Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426,"session_id":"97df21ca-d251-4713-bdca-5345fbc22e78"}
78882024-02-08T06:42:22.233Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:21.462485319Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426}
78892024-02-08T06:42:22.233ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78902024-02-08T06:42:22.233ZLoading write count information from file "/tmp/test_repair_verify.data"
78912024-02-08T06:42:22.234ZRun Repair workload
78922024-02-08T06:42:22.234Z01/30 Write block 476 len 28672 data: 5 4 4 4 4 4 5
78932024-02-08T06:42:22.234Z02/30 Read block 461 len 16384
78942024-02-08T06:42:22.234Z03/30 Read block 587 len 8192
78952024-02-08T06:42:22.234Z04/30 Write block 95 len 32768 data: 10 9 8 6 7 8 8 6
78962024-02-08T06:42:22.234Z05/30 Read block 135 len 24576
78972024-02-08T06:42:22.234Z06/30 Read block 299 len 20480
78982024-02-08T06:42:22.238Z07/30 Write block 166 len 16384 data: 11 11 9 10
78992024-02-08T06:42:22.238Z08/30 Read block 83 len 8192
79002024-02-08T06:42:22.238Z09/30 Write block 560 len 36864 data: 5 4 5 5 5 6 6 6 6
79012024-02-08T06:42:22.238Z10/30 Write block 515 len 12288 data: 3 5 5
79022024-02-08T06:42:22.238Z11/30 Read block 250 len 12288
79032024-02-08T06:42:22.238Z12/30 Flush
79042024-02-08T06:42:22.238Z13/30 Flush
79052024-02-08T06:42:22.238Z14/30 Write block 570 len 16384 data: 6 5 5 5
79062024-02-08T06:42:22.238Z15/30 Read block 425 len 28672
79072024-02-08T06:42:22.241Z16/30 Read block 320 len 8192
79082024-02-08T06:42:22.241Z17/30 Read block 557 len 20480
79092024-02-08T06:42:22.242Z18/30 Write block 314 len 36864 data: 5 3 4 4 6 8 11 9 10
79102024-02-08T06:42:22.242Z19/30 Read block 504 len 20480
79112024-02-08T06:42:22.245Z20/30 Read block 255 len 20480
79122024-02-08T06:42:22.245Z21/30 Read block 24 len 24576
79132024-02-08T06:42:22.245Z22/30 Read block 239 len 20480
79142024-02-08T06:42:22.245Z23/30 Read block 354 len 36864
79152024-02-08T06:42:22.248Z24/30 Write block 194 len 32768 data: 9 8 7 9 10 10 9 8
79162024-02-08T06:42:22.248Z25/30 Read block 193 len 40960
79172024-02-08T06:42:22.251Z26/30 Write block 235 len 16384 data: 7 6 5 6
79182024-02-08T06:42:22.251Z27/30 Write block 399 len 24576 data: 6 6 7 7 7 6
79192024-02-08T06:42:22.251Z28/30 Read block 219 len 36864
79202024-02-08T06:42:22.254Z29/30 Write block 593 len 28672 data: 7 7 5 5 4 2 3
79212024-02-08T06:42:22.254Z30/30 Write block 386 len 8192 data: 5 5
79222024-02-08T06:42:22.255Z----------------------------------------------------------------
79232024-02-08T06:42:22.255Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
79242024-02-08T06:42:22.255ZStates: Active Active Active
79252024-02-08T06:42:22.255ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
79262024-02-08T06:42:22.255Z 1 Acked 1000 Write 7 Sent Done Done false
79272024-02-08T06:42:22.255Z 2 Acked 1001 Read 4 Sent Done Done false
79282024-02-08T06:42:22.255Z 3 Acked 1002 Read 2 Sent Done Done false
79292024-02-08T06:42:22.255Z 4 Acked 1003 Write 8 Sent Done Done false
79302024-02-08T06:42:22.255Z 5 Acked 1004 Read 6 Sent Done Done false
79312024-02-08T06:42:22.255Z 6 Acked 1005 Read 5 Sent Done Done false
79322024-02-08T06:42:22.255Z 7 Acked 1006 Write 4 Sent Done Done false
79332024-02-08T06:42:22.255Z 8 Acked 1007 Read 2 Sent Done Done false
79342024-02-08T06:42:22.255Z 9 Acked 1008 Write 9 Sent Done Done false
79352024-02-08T06:42:22.255Z 10 Acked 1009 Write 3 Sent Done Done false
79362024-02-08T06:42:22.255Z 11 Acked 1010 Read 3 Sent Done Done false
79372024-02-08T06:42:22.256Z 12 Acked 1011 Flush 0 Sent Done Done false
79382024-02-08T06:42:22.256Z 13 Acked 1012 Flush 0 Sent Done Done false
79392024-02-08T06:42:22.256Z 14 Acked 1013 Write 4 Sent Done Done false
79402024-02-08T06:42:22.256Z 15 Acked 1014 Read 7 Sent Done Done false
79412024-02-08T06:42:22.256Z 16 Acked 1015 Read 2 Sent Done Done false
79422024-02-08T06:42:22.256Z 17 Acked 1016 Read 5 Sent Done Done false
79432024-02-08T06:42:22.256Z 18 Acked 1017 Write 9 Sent Done Done false
79442024-02-08T06:42:22.256Z 19 Acked 1018 Read 5 Sent Done Done false
79452024-02-08T06:42:22.256Z 20 Acked 1019 Read 5 Sent Done Done false
79462024-02-08T06:42:22.256Z 21 Acked 1020 Read 6 Sent Done Done false
79472024-02-08T06:42:22.256Z 22 Acked 1021 Read 5 Sent Done Done false
79482024-02-08T06:42:22.256Z 23 Acked 1022 Read 9 Sent Done Done false
79492024-02-08T06:42:22.256Z 24 Acked 1023 Write 8 Sent Done Done false
79502024-02-08T06:42:22.256Z 25 Acked 1024 Read 10 Sent Done Done false
79512024-02-08T06:42:22.256Z 26 Acked 1025 Write 4 Sent Done Done false
79522024-02-08T06:42:22.256Z 27 Acked 1026 Write 6 Sent Done Done false
79532024-02-08T06:42:22.257Z 28 Acked 1027 Read 9 Sent Done Done false
79542024-02-08T06:42:22.257Z 29 Acked 1028 Write 7 Sent Sent Sent false
79552024-02-08T06:42:22.257Z 30 Acked 1029 Write 2 Sent Sent Sent false
79562024-02-08T06:42:22.257Z STATES DS:0 DS:1 DS:2 TOTAL
79572024-02-08T06:42:22.257Z New 0 0 0 0
79582024-02-08T06:42:22.257Z Sent 30 2 2 34
79592024-02-08T06:42:22.257Z Done 0 28 28 56
79602024-02-08T06:42:22.257Z Skipped 0 0 0 0
79612024-02-08T06:42:22.257Z Error 0 0 0 0
79622024-02-08T06:42:22.257ZLast Flush: 0 1012 1012
79632024-02-08T06:42:22.257ZDownstairs last five completed:
79642024-02-08T06:42:22.257ZUpstairs last five completed: 30 29 28 27 26
79652024-02-08T06:42:22.257Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:21.491431895Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1426}
79662024-02-08T06:42:22.257ZWrote out file "/tmp/test_repair_verify.data"
79672024-02-08T06:42:22.264Z
79682024-02-08T06:42:22.264ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
79692024-02-08T06:42:22.278ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79702024-02-08T06:42:22.278Z 0 000-019 71 71 71 115 115 115 F F F
79712024-02-08T06:42:22.278Z 1 020-039 70 70 70 114 114 114 F F F
79722024-02-08T06:42:22.278Z 2 040-059 69 69 69 111 111 111 F F F
79732024-02-08T06:42:22.278Z 3 060-079 54 54 54 84 84 84 F F F
79742024-02-08T06:42:22.278Z 4 080-099 71 72 72 115 116 116 F F F <---
79752024-02-08T06:42:22.278Z 5 100-119 71 72 72 115 116 116 F F F <---
79762024-02-08T06:42:22.278Z 6 120-139 69 69 69 111 111 111 F F F
79772024-02-08T06:42:22.278Z 7 140-159 51 51 51 78 78 78 F F F
79782024-02-08T06:42:22.278Z 8 160-179 70 72 72 114 116 116 F F F <---
79792024-02-08T06:42:22.282Z 9 180-199 68 68 68 110 110 110 F T T <---
79802024-02-08T06:42:22.282Z 10 200-219 70 70 70 114 114 114 F T T <---
79812024-02-08T06:42:22.282Z 11 220-239 68 68 68 110 110 110 F T T <---
79822024-02-08T06:42:22.282Z 12 240-259 68 68 68 110 110 110 F F F
79832024-02-08T06:42:22.282Z 13 260-279 69 69 69 111 111 111 F F F
79842024-02-08T06:42:22.282Z 14 280-299 67 67 67 109 109 109 F F F
79852024-02-08T06:42:22.282Z 15 300-319 69 69 69 111 111 111 F T T <---
79862024-02-08T06:42:22.282Z 16 320-339 70 70 70 114 114 114 F T T <---
79872024-02-08T06:42:22.282Z 17 340-359 70 70 70 114 114 114 F F F
79882024-02-08T06:42:22.282Z 18 360-379 69 69 69 111 111 111 F F F
79892024-02-08T06:42:22.282Z 19 380-399 60 60 60 97 97 97 F T T <---
79902024-02-08T06:42:22.282Z 20 400-419 59 59 59 95 95 95 F T T <---
79912024-02-08T06:42:22.283Z 21 420-439 60 60 60 97 97 97 F F F
79922024-02-08T06:42:22.283Z 22 440-459 70 70 70 112 112 112 F F F
79932024-02-08T06:42:22.283Z 23 460-479 70 72 72 114 116 116 F F F <---
79942024-02-08T06:42:22.283Z 24 480-499 70 72 72 112 116 116 F F F <---
79952024-02-08T06:42:22.283Z 25 500-519 71 72 72 115 116 116 F F F <---
79962024-02-08T06:42:22.283Z 26 520-539 64 64 64 104 104 104 F F F
79972024-02-08T06:42:22.283Z 27 540-559 70 70 70 114 114 114 F F F
79982024-02-08T06:42:22.283Z 28 560-579 71 72 72 115 116 116 F T T <---
79992024-02-08T06:42:22.283Z 29 580-599 71 71 71 115 115 115 F T T <---
80002024-02-08T06:42:22.283ZMax gen: 72, Max flush: 116
80012024-02-08T06:42:22.283ZError: Difference in extent metadata found!
80022024-02-08T06:42:22.283ZA Difference in extent metadata is expected here
80032024-02-08T06:42:22.283ZOn loop 36
80042024-02-08T06:42:22.283Z
80052024-02-08T06:42:22.287ZVerifying data now
80062024-02-08T06:42:26.997ZNo change after verify
80072024-02-08T06:42:26.998ZLoop: 36 Downstairs dump after verify (and repair):
80082024-02-08T06:42:27.012ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80092024-02-08T06:42:27.013Z 0 000-019 71 71 71 115 115 115 F F F
80102024-02-08T06:42:27.013Z 1 020-039 70 70 70 114 114 114 F F F
80112024-02-08T06:42:27.013Z 2 040-059 69 69 69 111 111 111 F F F
80122024-02-08T06:42:27.013Z 3 060-079 54 54 54 84 84 84 F F F
80132024-02-08T06:42:27.013Z 4 080-099 72 72 72 116 116 116 F F F
80142024-02-08T06:42:27.013Z 5 100-119 72 72 72 116 116 116 F F F
80152024-02-08T06:42:27.016Z 6 120-139 69 69 69 111 111 111 F F F
80162024-02-08T06:42:27.016Z 7 140-159 51 51 51 78 78 78 F F F
80172024-02-08T06:42:27.016Z 8 160-179 72 72 72 116 116 116 F F F
80182024-02-08T06:42:27.016Z 9 180-199 73 73 73 117 117 117 F F F
80192024-02-08T06:42:27.017Z 10 200-219 73 73 73 117 117 117 F F F
80202024-02-08T06:42:27.017Z 11 220-239 73 73 73 117 117 117 F F F
80212024-02-08T06:42:27.017Z 12 240-259 68 68 68 110 110 110 F F F
80222024-02-08T06:42:27.017Z 13 260-279 69 69 69 111 111 111 F F F
80232024-02-08T06:42:27.017Z 14 280-299 67 67 67 109 109 109 F F F
80242024-02-08T06:42:27.017Z 15 300-319 73 73 73 117 117 117 F F F
80252024-02-08T06:42:27.017Z 16 320-339 73 73 73 117 117 117 F F F
80262024-02-08T06:42:27.017Z 17 340-359 70 70 70 114 114 114 F F F
80272024-02-08T06:42:27.018Z 18 360-379 69 69 69 111 111 111 F F F
80282024-02-08T06:42:27.018Z 19 380-399 73 73 73 117 117 117 F F F
80292024-02-08T06:42:27.018Z 20 400-419 73 73 73 117 117 117 F F F
80302024-02-08T06:42:27.018Z 21 420-439 60 60 60 97 97 97 F F F
80312024-02-08T06:42:27.018Z 22 440-459 70 70 70 112 112 112 F F F
80322024-02-08T06:42:27.018Z 23 460-479 72 72 72 116 116 116 F F F
80332024-02-08T06:42:27.018Z 24 480-499 72 72 72 116 116 116 F F F
80342024-02-08T06:42:27.019Z 25 500-519 72 72 72 116 116 116 F F F
80352024-02-08T06:42:27.019Z 26 520-539 64 64 64 104 104 104 F F F
80362024-02-08T06:42:27.019Z 27 540-559 70 70 70 114 114 114 F F F
80372024-02-08T06:42:27.019Z 28 560-579 73 73 73 117 117 117 F F F
80382024-02-08T06:42:27.019Z 29 580-599 73 73 73 117 117 117 F F F
80392024-02-08T06:42:27.019ZMax gen: 73, Max flush: 117
80402024-02-08T06:42:27.019Z
80412024-02-08T06:42:27.019ZBegin loop 37
80422024-02-08T06:42:27.020ZDownstairs to restart: 1
80432024-02-08T06:42:27.033Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.271548875Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434}
80442024-02-08T06:42:27.033Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",Crucible runtime is spawned
80452024-02-08T06:42:27.038Z\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.271849612Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434}
80462024-02-08T06:42:27.038Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.271896968Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434}
80472024-02-08T06:42:27.038Z{"msg":"Crucible 4e908179-e8c1-42cf-9448-e04431dc96e9 has session id: 80db02f8-2212-49eb-a766-59db30921029","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.271926976Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80482024-02-08T06:42:27.038Z{"msg":"Upstairs opts: Upstairs UUID: 4e908179-e8c1-42cf-9448-e04431dc96e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.271971193Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80492024-02-08T06:42:27.039Z{"msg":"Crucible stats registered with UUID: 4e908179-e8c1-42cf-9448-e04431dc96e9","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272021939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80502024-02-08T06:42:27.039Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272051177Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434}
80512024-02-08T06:42:27.039Z{"msg":"4e908179-e8c1-42cf-9448-e04431dc96e9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272090934Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80522024-02-08T06:42:27.039Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272133591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80532024-02-08T06:42:27.039Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272186977Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80542024-02-08T06:42:27.039Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272236803Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80552024-02-08T06:42:27.039Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.27228036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80562024-02-08T06:42:27.039Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272326436Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80572024-02-08T06:42:27.039Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272395661Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80582024-02-08T06:42:27.039Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272439538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"io task","client":"2","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80592024-02-08T06:42:27.039Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272466386Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"io task","client":"1","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80602024-02-08T06:42:27.039Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272490124Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"io task","client":"0","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80612024-02-08T06:42:27.039Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272512672Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"io task","client":"2","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80622024-02-08T06:42:27.039Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.27253475Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"io task","client":"1","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80632024-02-08T06:42:27.039Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.272558969Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"io task","client":"0","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80642024-02-08T06:42:27.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.273082559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80652024-02-08T06:42:27.040Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.273120526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80662024-02-08T06:42:27.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.273140225Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80672024-02-08T06:42:27.040Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.273177602Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80682024-02-08T06:42:27.040Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.273697363Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80692024-02-08T06:42:27.040Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.273753169Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80702024-02-08T06:42:27.040Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.274311157Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80712024-02-08T06:42:27.040Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.274354074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80722024-02-08T06:42:27.040Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.274383532Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80732024-02-08T06:42:27.040Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.27441068Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80742024-02-08T06:42:27.040Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.274435338Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80752024-02-08T06:42:27.040Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.274460786Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80762024-02-08T06:42:27.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.274488344Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80772024-02-08T06:42:27.040Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.274512482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80782024-02-08T06:42:27.040Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.274948889Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80792024-02-08T06:42:27.045Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275441142Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80802024-02-08T06:42:27.045Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275478089Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80812024-02-08T06:42:27.045Z{"msg":"[0]R flush_numbers[0..12]: [115, 114, 111, 84, 116, 116, 111, 78, 116, 117, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275494428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80822024-02-08T06:42:27.045Z{"msg":"[0]R generation[0..12]: [71, 70, 69, 54, 72, 72, 69, 51, 72, 73, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275509127Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80832024-02-08T06:42:27.045Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275533215Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80842024-02-08T06:42:27.046Z{"msg":"[1]R flush_numbers[0..12]: [115, 114, 111, 84, 116, 116, 111, 78, 116, 117, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275553364Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80852024-02-08T06:42:27.046Z{"msg":"[1]R generation[0..12]: [71, 70, 69, 54, 72, 72, 69, 51, 72, 73, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275574582Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80862024-02-08T06:42:27.046Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275588911Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80872024-02-08T06:42:27.046Z{"msg":"[2]R flush_numbers[0..12]: [115, 114, 111, 84, 116, 116, 111, 78, 116, 117, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.27560379Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80882024-02-08T06:42:27.046Z{"msg":"[2]R generation[0..12]: [71, 70, 69, 54, 72, 72, 69, 51, 72, 73, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275624889Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80892024-02-08T06:42:27.046Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275639237Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80902024-02-08T06:42:27.046Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275658946Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80912024-02-08T06:42:27.046Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
80922024-02-08T06:42:27.046Z2024-02-08T06:42:26.275672515Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80932024-02-08T06:42:27.046Z{"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275698043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80942024-02-08T06:42:27.046Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275717982Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80952024-02-08T06:42:27.046Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.27573177Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80962024-02-08T06:42:27.046Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275750409Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80972024-02-08T06:42:27.046Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275764428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80982024-02-08T06:42:27.047Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275802975Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"80db02f8-2212-49eb-a766-59db30921029"}
80992024-02-08T06:42:27.047Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275823284Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
81002024-02-08T06:42:27.047Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275844972Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
81012024-02-08T06:42:27.047Z{"msg":"4e908179-e8c1-42cf-9448-e04431dc96e9 is now active with session: 80db02f8-2212-49eb-a766-59db30921029","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.27586883Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
81022024-02-08T06:42:27.047Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275899438Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434,"session_id":"80db02f8-2212-49eb-a766-59db30921029"}
81032024-02-08T06:42:27.047Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:26.275922556Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434}
81042024-02-08T06:42:27.047ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
81052024-02-08T06:42:27.047ZLoading write count information from file "/tmp/test_repair_verify.data"
81062024-02-08T06:42:27.047ZRun Repair workload
81072024-02-08T06:42:27.047Z01/30 Write block 100 len 40960 data: 9 9 7 8 8 5 6 6 7 5
81082024-02-08T06:42:27.047Z02/30 Read block 385 len 24576
81092024-02-08T06:42:27.047Z03/30 Write block 28 len 8192 data: 7 6
81102024-02-08T06:42:27.047Z04/30 Write block 7 len 16384 data: 7 7 6 7
81112024-02-08T06:42:27.047Z05/30 Write block 334 len 40960 data: 10 9 11 10 11 11 10 9 8 6
81122024-02-08T06:42:27.047Z06/30 Read block 19 len 24576
81132024-02-08T06:42:27.051Z07/30 Write block 589 len 40960 data: 9 9 8 7 8 8 6 6 5 3
81142024-02-08T06:42:27.051Z08/30 Write block 182 len 4096 data: 9
81152024-02-08T06:42:27.051Z09/30 Read block 114 len 32768
81162024-02-08T06:42:27.051Z10/30 Read block 436 len 8192
81172024-02-08T06:42:27.051Z11/30 Flush
81182024-02-08T06:42:27.051Z12/30 Write block 429 len 16384 data: 4 5 4 4
81192024-02-08T06:42:27.051Z13/30 Write block 57 len 16384 data: 8 4 4 5
81202024-02-08T06:42:27.054Z14/30 Write block 391 len 8192 data: 5 5
81212024-02-08T06:42:27.054Z15/30 Write block 80 len 24576 data: 5 4 5 6 6 6
81222024-02-08T06:42:27.054Z16/30 Write block 466 len 24576 data: 4 6 6 7 7 6
81232024-02-08T06:42:27.055Z17/30 Write block 348 len 4096 data: 5
81242024-02-08T06:42:27.055Z18/30 Read block 189 len 40960
81252024-02-08T06:42:27.057Z19/30 Write block 547 len 40960 data: 7 7 10 8 9 8 8 8 7 10
81262024-02-08T06:42:27.058Z20/30 Read block 71 len 28672
81272024-02-08T06:42:27.058Z21/30 Write block 24 len 36864 data: 9 10 9 8 8 7 7 6 6
81282024-02-08T06:42:27.061Z22/30 Write block 255 len 12288 data: 7 7 6
81292024-02-08T06:42:27.061Z23/30 Read block 562 len 4096
81302024-02-08T06:42:27.061Z24/30 Read block 65 len 32768
81312024-02-08T06:42:27.061Z25/30 Read block 169 len 36864
81322024-02-08T06:42:27.064Z26/30 Write block 401 len 36864 data: 8 8 8 7 7 6 6 5 4
81332024-02-08T06:42:27.064Z27/30 Read block 217 len 32768
81342024-02-08T06:42:27.064Z28/30 Read block 376 len 28672
81352024-02-08T06:42:27.068Z29/30 Read block 35 len 20480
81362024-02-08T06:42:27.068Z30/30 Read block 224 len 16384
81372024-02-08T06:42:27.068Z----------------------------------------------------------------
81382024-02-08T06:42:27.068Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
81392024-02-08T06:42:27.068ZStates: Active Active Active
81402024-02-08T06:42:27.068ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
81412024-02-08T06:42:27.068Z 1 Acked 1000 Write 10 Done Sent Done false
81422024-02-08T06:42:27.068Z 2 Acked 1001 Read 6 Done Sent Done false
81432024-02-08T06:42:27.068Z 3 Acked 1002 Write 2 Done Sent Done false
81442024-02-08T06:42:27.068Z 4 Acked 1003 Write 4 Done Sent Done false
81452024-02-08T06:42:27.068Z 5 Acked 1004 Write 10 Done Sent Done false
81462024-02-08T06:42:27.068Z 6 Acked 1005 Read 6 Done Sent Done false
81472024-02-08T06:42:27.069Z 7 Acked 1006 Write 10 Done Sent Done false
81482024-02-08T06:42:27.069Z 8 Acked 1007 Write 1 Done Sent Done false
81492024-02-08T06:42:27.069Z 9 Acked 1008 Read 8 Done Sent Done false
81502024-02-08T06:42:27.069Z 10 Acked 1009 Read 2 Done Sent Done false
81512024-02-08T06:42:27.069Z 11 Acked 1010 Flush 0 Done Sent Done false
81522024-02-08T06:42:27.069Z 12 Acked 1011 Write 4 Done Sent Done false
81532024-02-08T06:42:27.069Z 13 Acked 1012 Write 4 Done Sent Done false
81542024-02-08T06:42:27.069Z 14 Acked 1013 Write 2 Done Sent Done false
81552024-02-08T06:42:27.069Z 15 Acked 1014 Write 6 Done Sent Done false
81562024-02-08T06:42:27.069Z 16 Acked 1015 Write 6 Done Sent Done false
81572024-02-08T06:42:27.069Z 17 Acked 1016 Write 1 Done Sent Done false
81582024-02-08T06:42:27.069Z 18 Acked 1017 Read 10 Done Sent Done false
81592024-02-08T06:42:27.069Z 19 Acked 1018 Write 10 Done Sent Done false
81602024-02-08T06:42:27.069Z 20 Acked 1019 Read 7 Done Sent Done false
81612024-02-08T06:42:27.069Z 21 Acked 1020 Write 9 Done Sent Done false
81622024-02-08T06:42:27.069Z 22 Acked 1021 Write 3 Done Sent Done false
81632024-02-08T06:42:27.070Z 23 Acked 1022 Read 1 Done Sent Done false
81642024-02-08T06:42:27.070Z 24 Acked 1023 Read 8 Done Sent Done false
81652024-02-08T06:42:27.070Z 25 Acked 1024 Read 9 Done Sent Done false
81662024-02-08T06:42:27.070Z 26 Acked 1025 Write 9 Done Sent Done false
81672024-02-08T06:42:27.070Z 27 Acked 1026 Read 8 Done Sent Done false
81682024-02-08T06:42:27.070Z 28 Acked 1027 Read 7 Done Sent Done false
81692024-02-08T06:42:27.070Z 29 Acked 1028 Read 5 Done Sent Done false
81702024-02-08T06:42:27.070Z 30 Acked 1029 Read 4 Done Sent Done false
81712024-02-08T06:42:27.070Z STATES DS:0 DS:1 DS:2 TOTAL
81722024-02-08T06:42:27.070Z New 0 0 0 0
81732024-02-08T06:42:27.070Z Sent 0 30 0 30
81742024-02-08T06:42:27.070Z Done 30 0 30 60
81752024-02-08T06:42:27.070Z Skipped 0 0 0 0
81762024-02-08T06:42:27.070Z Error 0 0 0 0
81772024-02-08T06:42:27.070ZLast Flush: 1010 0 1010
81782024-02-08T06:42:27.071ZDownstairs last five completed:
81792024-02-08T06:42:27.071ZUpstairs last five completed: 30 29 28 27 26
81802024-02-08T06:42:27.071Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:26.304973525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1434}
81812024-02-08T06:42:27.071ZWrote out file "/tmp/test_repair_verify.data"
81822024-02-08T06:42:27.074Z
81832024-02-08T06:42:27.074ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
81842024-02-08T06:42:27.092ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81852024-02-08T06:42:27.092Z 0 000-019 74 71 74 118 115 118 F F F <---
81862024-02-08T06:42:27.092Z 1 020-039 74 70 74 118 114 118 T F T <---
81872024-02-08T06:42:27.092Z 2 040-059 69 69 69 111 111 111 T F T <---
81882024-02-08T06:42:27.092Z 3 060-079 54 54 54 84 84 84 T F T <---
81892024-02-08T06:42:27.092Z 4 080-099 72 72 72 116 116 116 T F T <---
81902024-02-08T06:42:27.092Z 5 100-119 74 72 74 118 116 118 F F F <---
81912024-02-08T06:42:27.092Z 6 120-139 69 69 69 111 111 111 F F F
81922024-02-08T06:42:27.092Z 7 140-159 51 51 51 78 78 78 F F F
81932024-02-08T06:42:27.095Z 8 160-179 72 72 72 116 116 116 F F F
81942024-02-08T06:42:27.096Z 9 180-199 74 73 74 118 117 118 F F F <---
81952024-02-08T06:42:27.096Z 10 200-219 73 73 73 117 117 117 F F F
81962024-02-08T06:42:27.096Z 11 220-239 73 73 73 117 117 117 F F F
81972024-02-08T06:42:27.096Z 12 240-259 68 68 68 110 110 110 T F T <---
81982024-02-08T06:42:27.096Z 13 260-279 69 69 69 111 111 111 F F F
81992024-02-08T06:42:27.096Z 14 280-299 67 67 67 109 109 109 F F F
82002024-02-08T06:42:27.096Z 15 300-319 73 73 73 117 117 117 F F F
82012024-02-08T06:42:27.096Z 16 320-339 74 73 74 118 117 118 F F F <---
82022024-02-08T06:42:27.096Z 17 340-359 74 70 74 118 114 118 T F T <---
82032024-02-08T06:42:27.096Z 18 360-379 69 69 69 111 111 111 F F F
82042024-02-08T06:42:27.096Z 19 380-399 73 73 73 117 117 117 T F T <---
82052024-02-08T06:42:27.096Z 20 400-419 73 73 73 117 117 117 T F T <---
82062024-02-08T06:42:27.096Z 21 420-439 60 60 60 97 97 97 T F T <---
82072024-02-08T06:42:27.096Z 22 440-459 70 70 70 112 112 112 F F F
82082024-02-08T06:42:27.097Z 23 460-479 72 72 72 116 116 116 T F T <---
82092024-02-08T06:42:27.097Z 24 480-499 72 72 72 116 116 116 F F F
82102024-02-08T06:42:27.097Z 25 500-519 72 72 72 116 116 116 F F F
82112024-02-08T06:42:27.097Z 26 520-539 64 64 64 104 104 104 F F F
82122024-02-08T06:42:27.097Z 27 540-559 70 70 70 114 114 114 T F T <---
82132024-02-08T06:42:27.097Z 28 560-579 73 73 73 117 117 117 F F F
82142024-02-08T06:42:27.097Z 29 580-599 74 73 74 118 117 118 F F F <---
82152024-02-08T06:42:27.097ZMax gen: 74, Max flush: 118
82162024-02-08T06:42:27.097ZError: Difference in extent metadata found!
82172024-02-08T06:42:27.097ZA Difference in extent metadata is expected here
82182024-02-08T06:42:27.097ZOn loop 37
82192024-02-08T06:42:27.097Z
82202024-02-08T06:42:27.100ZVerifying data now
82212024-02-08T06:42:31.869ZNo change after verify
82222024-02-08T06:42:31.869ZLoop: 37 Downstairs dump after verify (and repair):
82232024-02-08T06:42:31.884ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
82242024-02-08T06:42:31.884Z 0 000-019 74 74 74 118 118 118 F F F
82252024-02-08T06:42:31.884Z 1 020-039 75 75 75 119 119 119 F F F
82262024-02-08T06:42:31.884Z 2 040-059 75 75 75 119 119 119 F F F
82272024-02-08T06:42:31.884Z 3 060-079 75 75 75 119 119 119 F F F
82282024-02-08T06:42:31.884Z 4 080-099 75 75 75 119 119 119 F F F
82292024-02-08T06:42:31.884Z 5 100-119 74 74 74 118 118 118 F F F
82302024-02-08T06:42:31.884Z 6 120-139 69 69 69 111 111 111 F F F
82312024-02-08T06:42:31.885Z 7 140-159 51 51 51 78 78 78 F F F
82322024-02-08T06:42:31.885Z 8 160-179 72 72 72 116 116 116 F F F
82332024-02-08T06:42:31.885Z 9 180-199 74 74 74 118 118 118 F F F
82342024-02-08T06:42:31.885Z 10 200-219 73 73 73 117 117 117 F F F
82352024-02-08T06:42:31.885Z 11 220-239 73 73 73 117 117 117 F F F
82362024-02-08T06:42:31.890Z 12 240-259 75 75 75 119 119 119 F F F
82372024-02-08T06:42:31.890Z 13 260-279 69 69 69 111 111 111 F F F
82382024-02-08T06:42:31.890Z 14 280-299 67 67 67 109 109 109 F F F
82392024-02-08T06:42:31.890Z 15 300-319 73 73 73 117 117 117 F F F
82402024-02-08T06:42:31.890Z 16 320-339 74 74 74 118 118 118 F F F
82412024-02-08T06:42:31.890Z 17 340-359 75 75 75 119 119 119 F F F
82422024-02-08T06:42:31.890Z 18 360-379 69 69 69 111 111 111 F F F
82432024-02-08T06:42:31.890Z 19 380-399 75 75 75 119 119 119 F F F
82442024-02-08T06:42:31.890Z 20 400-419 75 75 75 119 119 119 F F F
82452024-02-08T06:42:31.890Z 21 420-439 75 75 75 119 119 119 F F F
82462024-02-08T06:42:31.890Z 22 440-459 70 70 70 112 112 112 F F F
82472024-02-08T06:42:31.890Z 23 460-479 75 75 75 119 119 119 F F F
82482024-02-08T06:42:31.891Z 24 480-499 72 72 72 116 116 116 F F F
82492024-02-08T06:42:31.891Z 25 500-519 72 72 72 116 116 116 F F F
82502024-02-08T06:42:31.891Z 26 520-539 64 64 64 104 104 104 F F F
82512024-02-08T06:42:31.891Z 27 540-559 75 75 75 119 119 119 F F F
82522024-02-08T06:42:31.891Z 28 560-579 73 73 73 117 117 117 F F F
82532024-02-08T06:42:31.891Z 29 580-599 74 74 74 118 118 118 F F F
82542024-02-08T06:42:31.891ZMax gen: 75, Max flush: 119
82552024-02-08T06:42:31.891Z
82562024-02-08T06:42:31.891ZBegin loop 38
82572024-02-08T06:42:31.891ZDownstairs to restart: 2
82582024-02-08T06:42:31.905Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.142738536Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442}
82592024-02-08T06:42:31.905Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.142974108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442}
82602024-02-08T06:42:31.905Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.142992447Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442}
82612024-02-08T06:42:31.910Z{"msg":"Crucible 36320a36-3657-423b-bff4-c01d3f7e720f has session id: 9b813d26-9f35-4cd5-8749-acf930954c24","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143025184Z","hostname":"Crucible runtime is spawned
82622024-02-08T06:42:31.910Zip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82632024-02-08T06:42:31.910Z{"msg":"Upstairs opts: Upstairs UUID: 36320a36-3657-423b-bff4-c01d3f7e720f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143066381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82642024-02-08T06:42:31.910Z{"msg":"Crucible stats registered with UUID: 36320a36-3657-423b-bff4-c01d3f7e720f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143111568Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82652024-02-08T06:42:31.911Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143128767Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442}
82662024-02-08T06:42:31.911Z{"msg":"36320a36-3657-423b-bff4-c01d3f7e720f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143152825Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82672024-02-08T06:42:31.911Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143166784Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82682024-02-08T06:42:31.911Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143201381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82692024-02-08T06:42:31.911Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.14322384Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82702024-02-08T06:42:31.911Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143253627Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82712024-02-08T06:42:31.911Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143279405Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82722024-02-08T06:42:31.911Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143320342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82732024-02-08T06:42:31.911Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.1433555Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"io task","client":"1","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82742024-02-08T06:42:31.911Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143382368Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"io task","client":"0","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82752024-02-08T06:42:31.911Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143406496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"io task","client":"2","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82762024-02-08T06:42:31.911Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143557675Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"io task","client":"0","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82772024-02-08T06:42:31.911Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143608341Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"io task","client":"2","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82782024-02-08T06:42:31.911Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.143638889Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"io task","client":"1","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82792024-02-08T06:42:31.912Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.144274601Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82802024-02-08T06:42:31.912Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.144315138Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82812024-02-08T06:42:31.912Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.144377903Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82822024-02-08T06:42:31.912Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.14442161Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82832024-02-08T06:42:31.912Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.145002336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82842024-02-08T06:42:31.912Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.145113358Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82852024-02-08T06:42:31.912Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.145475891Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82862024-02-08T06:42:31.912Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.145513878Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82872024-02-08T06:42:31.912Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.145539016Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82882024-02-08T06:42:31.912Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.14575665Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82892024-02-08T06:42:31.912Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.145792837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82902024-02-08T06:42:31.912Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.145808726Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82912024-02-08T06:42:31.912Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.145828894Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82922024-02-08T06:42:31.912Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.145843363Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82932024-02-08T06:42:31.912Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.14641442Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82942024-02-08T06:42:31.920Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.146893094Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82952024-02-08T06:42:31.920Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.14694626Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82962024-02-08T06:42:31.920Z{"msg":"[0]R flush_numbers[0..12]: [118, 119, 119, 119, 119, 118, 111, 78, 116, 118, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.146985227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82972024-02-08T06:42:31.920Z{"msg":"[0]R generation[0..12]: [74, 75, 75, 75, 75, 74, 69, 51, 72, 74, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147003656Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82982024-02-08T06:42:31.920Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147019555Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
82992024-02-08T06:42:31.920Z{"msg":"[1]R flush_numbers[0..12]: [118, 119, 119, 119, 119, 118, 111, 78, 116, 118, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147033923Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83002024-02-08T06:42:31.920Z{"msg":"[1]R generation[0..12]: [74, 75, 75, 75, 75, 74, 69, 51, 72, 74, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147048633Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83012024-02-08T06:42:31.920Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
83022024-02-08T06:42:31.920Z,"time":"2024-02-08T06:42:31.147072211Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83032024-02-08T06:42:31.920Z{"msg":"[2]R flush_numbers[0..12]: [118, 119, 119, 119, 119, 118, 111, 78, 116, 118, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.14708749Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83042024-02-08T06:42:31.921Z{"msg":"[2]R generation[0..12]: [74, 75, 75, 75, 75, 74, 69, 51, 72, 74, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147106878Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83052024-02-08T06:42:31.921Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147141446Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83062024-02-08T06:42:31.921Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147165894Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83072024-02-08T06:42:31.921Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147201161Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83082024-02-08T06:42:31.921Z{"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147236708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83092024-02-08T06:42:31.921Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147261966Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83102024-02-08T06:42:31.921Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147292994Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83112024-02-08T06:42:31.921Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147315712Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83122024-02-08T06:42:31.921Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147335361Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
83132024-02-08T06:42:31.921Zip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83142024-02-08T06:42:31.921Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147359939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83152024-02-08T06:42:31.921Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147376448Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83162024-02-08T06:42:31.921Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147390417Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83172024-02-08T06:42:31.921Z{"msg":"36320a36-3657-423b-bff4-c01d3f7e720f is now active with session: 9b813d26-9f35-4cd5-8749-acf930954c24","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147403936Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83182024-02-08T06:42:31.921Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147424514Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442,"session_id":"9b813d26-9f35-4cd5-8749-acf930954c24"}
83192024-02-08T06:42:31.922Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:31.147437203Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442}
83202024-02-08T06:42:31.922ZLoading write count information from file "/tmp/test_repair_verify.data"
83212024-02-08T06:42:31.922ZRun Repair workload
83222024-02-08T06:42:31.922Z01/30 Flush
83232024-02-08T06:42:31.922Z02/30 Write block 456 len 16384 data: 9 9 8 8
83242024-02-08T06:42:31.922Z03/30 Read block 252 len 36864
83252024-02-08T06:42:31.922Z04/30 Read block 405 len 36864
83262024-02-08T06:42:31.922Z05/30 Read block 175 len 40960
83272024-02-08T06:42:31.922Z06/30 Write block 137 len 16384 data: 8 6 5 4
83282024-02-08T06:42:31.922Z07/30 Write block 255 len 16384 data: 8 8 7 5
83292024-02-08T06:42:31.926Z08/30 Write block 498 len 16384 data: 5 6 6 6
83302024-02-08T06:42:31.926Z09/30 Write block 531 len 24576 data: 9 9 9 9 9 10
83312024-02-08T06:42:31.926Z10/30 Read block 382 len 8192
83322024-02-08T06:42:31.926Z11/30 Write block 491 len 40960 data: 7 6 6 6 6 5 5 6 7 7
83332024-02-08T06:42:31.926Z12/30 Read block 484 len 20480
83342024-02-08T06:42:31.926Z13/30 Read block 408 len 36864
83352024-02-08T06:42:31.926Z14/30 Write block 138 len 16384 data: 7 6 5 5
83362024-02-08T06:42:31.926Z15/30 Write block 68 len 8192 data: 7 7
83372024-02-08T06:42:31.926Z16/30 Read block 203 len 24576
83382024-02-08T06:42:31.926Z17/30 Read block 213 len 12288
83392024-02-08T06:42:31.926Z18/30 Read block 474 len 8192
83402024-02-08T06:42:31.926Z19/30 Read block 48 len 40960
83412024-02-08T06:42:31.929Z20/30 Write block 52 len 20480 data: 5 5 7 7 9
83422024-02-08T06:42:31.929Z21/30 Write block 160 len 32768 data: 5 5 8 9 10 11 12 12
83432024-02-08T06:42:31.930Z22/30 Read block 131 len 24576
83442024-02-08T06:42:31.930Z23/30 Read block 541 len 24576
83452024-02-08T06:42:31.933Z24/30 Write block 362 len 16384 data: 5 4 5 5
83462024-02-08T06:42:31.933Z25/30 Read block 440 len 20480
83472024-02-08T06:42:31.933Z26/30 Read block 516 len 4096
83482024-02-08T06:42:31.933Z27/30 Read block 174 len 40960
83492024-02-08T06:42:31.937Z28/30 Read block 584 len 28672
83502024-02-08T06:42:31.937Z29/30 Read block 481 len 12288
83512024-02-08T06:42:31.937Z30/30 Write block 127 len 32768 data: 2 4 4 5 7 7 7 5
83522024-02-08T06:42:31.937Z----------------------------------------------------------------
83532024-02-08T06:42:31.937Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
83542024-02-08T06:42:31.937ZStates: Active Active Active
83552024-02-08T06:42:31.937ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
83562024-02-08T06:42:31.937Z 1 Acked 1000 Flush 0 Done Done Sent false
83572024-02-08T06:42:31.937Z 2 Acked 1001 Write 4 Done Done Sent false
83582024-02-08T06:42:31.937Z 3 Acked 1002 Read 9 Done Done Sent false
83592024-02-08T06:42:31.937Z 4 Acked 1003 Read 9 Done Done Sent false
83602024-02-08T06:42:31.938Z 5 Acked 1004 Read 10 Done Done Sent false
83612024-02-08T06:42:31.938Z 6 Acked 1005 Write 4 Done Done Sent false
83622024-02-08T06:42:31.938Z 7 Acked 1006 Write 4 Done Done Sent false
83632024-02-08T06:42:31.938Z 8 Acked 1007 Write 4 Done Done Sent false
83642024-02-08T06:42:31.938Z 9 Acked 1008 Write 6 Done Done Sent false
83652024-02-08T06:42:31.938Z 10 Acked 1009 Read 2 Done Done Sent false
83662024-02-08T06:42:31.938Z 11 Acked 1010 Write 10 Done Done Sent false
83672024-02-08T06:42:31.938Z 12 Acked 1011 Read 5 Done Done Sent false
83682024-02-08T06:42:31.938Z 13 Acked 1012 Read 9 Done Done Sent false
83692024-02-08T06:42:31.938Z 14 Acked 1013 Write 4 Done Done Sent false
83702024-02-08T06:42:31.938Z 15 Acked 1014 Write 2 Done Done Sent false
83712024-02-08T06:42:31.938Z 16 Acked 1015 Read 6 Done Done Sent false
83722024-02-08T06:42:31.938Z 17 Acked 1016 Read 3 Done Done Sent false
83732024-02-08T06:42:31.938Z 18 Acked 1017 Read 2 Done Done Sent false
83742024-02-08T06:42:31.938Z 19 Acked 1018 Read 10 Done Done Sent false
83752024-02-08T06:42:31.938Z 20 Acked 1019 Write 5 Done Done Sent false
83762024-02-08T06:42:31.939Z 21 Acked 1020 Write 8 Done Done Sent false
83772024-02-08T06:42:31.939Z 22 Acked 1021 Read 6 Done Done Sent false
83782024-02-08T06:42:31.939Z 23 Acked 1022 Read 6 Done Done Sent false
83792024-02-08T06:42:31.939Z 24 Acked 1023 Write 4 Done Done Sent false
83802024-02-08T06:42:31.939Z 25 Acked 1024 Read 5 Done Done Sent false
83812024-02-08T06:42:31.939Z 26 Acked 1025 Read 1 Done Done Sent false
83822024-02-08T06:42:31.939Z 27 Acked 1026 Read 10 Done Done Sent false
83832024-02-08T06:42:31.939Z 28 Acked 1027 Read 7 Done Done Sent false
83842024-02-08T06:42:31.939Z 29 Acked 1028 Read 3 Done Done Sent false
83852024-02-08T06:42:31.939Z 30 Acked 1029 Write 8 Sent Sent Sent false
83862024-02-08T06:42:31.939Z STATES DS:0 DS:1 DS:2 TOTAL
83872024-02-08T06:42:31.939Z New 0 0 0 0
83882024-02-08T06:42:31.939Z Sent 1 1 30 32
83892024-02-08T06:42:31.939Z Done 29 29 0 58
83902024-02-08T06:42:31.939Z Skipped 0 0 0 0
83912024-02-08T06:42:31.940Z Error 0 0 0 0
83922024-02-08T06:42:31.940ZLast Flush: 1000 1000 0
83932024-02-08T06:42:31.940ZDownstairs last five completed:
83942024-02-08T06:42:31.940ZUpstairs last five completed: 30 29 28 27 26
83952024-02-08T06:42:31.940Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:31.174091782Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1442}
83962024-02-08T06:42:31.940ZWrote out file "/tmp/test_repair_verify.data"
83972024-02-08T06:42:31.943Z
83982024-02-08T06:42:31.943ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83992024-02-08T06:42:31.961ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
84002024-02-08T06:42:31.961Z 0 000-019 74 74 74 118 118 118 F F F
84012024-02-08T06:42:31.961Z 1 020-039 75 75 75 119 119 119 F F F
84022024-02-08T06:42:31.961Z 2 040-059 75 75 75 119 119 119 T T F <---
84032024-02-08T06:42:31.962Z 3 060-079 75 75 75 119 119 119 T T F <---
84042024-02-08T06:42:31.962Z 4 080-099 75 75 75 119 119 119 F F F
84052024-02-08T06:42:31.965Z 5 100-119 74 74 74 118 118 118 F F F
84062024-02-08T06:42:31.965Z 6 120-139 69 69 69 111 111 111 T T F <---
84072024-02-08T06:42:31.965Z 7 140-159 51 51 51 78 78 78 T T F <---
84082024-02-08T06:42:31.965Z 8 160-179 72 72 72 116 116 116 T T F <---
84092024-02-08T06:42:31.965Z 9 180-199 74 74 74 118 118 118 F F F
84102024-02-08T06:42:31.965Z 10 200-219 73 73 73 117 117 117 F F F
84112024-02-08T06:42:31.965Z 11 220-239 73 73 73 117 117 117 F F F
84122024-02-08T06:42:31.965Z 12 240-259 75 75 75 119 119 119 T T F <---
84132024-02-08T06:42:31.965Z 13 260-279 69 69 69 111 111 111 F F F
84142024-02-08T06:42:31.966Z 14 280-299 67 67 67 109 109 109 F F F
84152024-02-08T06:42:31.966Z 15 300-319 73 73 73 117 117 117 F F F
84162024-02-08T06:42:31.966Z 16 320-339 74 74 74 118 118 118 F F F
84172024-02-08T06:42:31.966Z 17 340-359 75 75 75 119 119 119 F F F
84182024-02-08T06:42:31.966Z 18 360-379 69 69 69 111 111 111 T T F <---
84192024-02-08T06:42:31.966Z 19 380-399 75 75 75 119 119 119 F F F
84202024-02-08T06:42:31.966Z 20 400-419 75 75 75 119 119 119 F F F
84212024-02-08T06:42:31.966Z 21 420-439 75 75 75 119 119 119 F F F
84222024-02-08T06:42:31.966Z 22 440-459 70 70 70 112 112 112 T T F <---
84232024-02-08T06:42:31.966Z 23 460-479 75 75 75 119 119 119 F F F
84242024-02-08T06:42:31.966Z 24 480-499 72 72 72 116 116 116 T T F <---
84252024-02-08T06:42:31.966Z 25 500-519 72 72 72 116 116 116 T T F <---
84262024-02-08T06:42:31.966Z 26 520-539 64 64 64 104 104 104 T T F <---
84272024-02-08T06:42:31.966Z 27 540-559 75 75 75 119 119 119 F F F
84282024-02-08T06:42:31.966Z 28 560-579 73 73 73 117 117 117 F F F
84292024-02-08T06:42:31.966Z 29 580-599 74 74 74 118 118 118 F F F
84302024-02-08T06:42:31.967ZMax gen: 75, Max flush: 119
84312024-02-08T06:42:31.967ZError: Difference in extent metadata found!
84322024-02-08T06:42:31.967ZA Difference in extent metadata is expected here
84332024-02-08T06:42:31.967ZOn loop 38
84342024-02-08T06:42:31.967Z
84352024-02-08T06:42:31.970ZVerifying data now
84362024-02-08T06:42:36.537ZNo change after verify
84372024-02-08T06:42:36.538ZLoop: 38 Downstairs dump after verify (and repair):
84382024-02-08T06:42:36.553ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
84392024-02-08T06:42:36.553Z 0 000-019 74 74 74 118 118 118 F F F
84402024-02-08T06:42:36.553Z 1 020-039 75 75 75 119 119 119 F F F
84412024-02-08T06:42:36.553Z 2 040-059 76 76 76 120 120 120 F F F
84422024-02-08T06:42:36.553Z 3 060-079 76 76 76 120 120 120 F F F
84432024-02-08T06:42:36.553Z 4 080-099 75 75 75 119 119 119 F F F
84442024-02-08T06:42:36.553Z 5 100-119 74 74 74 118 118 118 F F F
84452024-02-08T06:42:36.553Z 6 120-139 76 76 76 120 120 120 F F F
84462024-02-08T06:42:36.553Z 7 140-159 76 76 76 120 120 120 F F F
84472024-02-08T06:42:36.557Z 8 160-179 76 76 76 120 120 120 F F F
84482024-02-08T06:42:36.557Z 9 180-199 74 74 74 118 118 118 F F F
84492024-02-08T06:42:36.557Z 10 200-219 73 73 73 117 117 117 F F F
84502024-02-08T06:42:36.557Z 11 220-239 73 73 73 117 117 117 F F F
84512024-02-08T06:42:36.557Z 12 240-259 76 76 76 120 120 120 F F F
84522024-02-08T06:42:36.557Z 13 260-279 69 69 69 111 111 111 F F F
84532024-02-08T06:42:36.557Z 14 280-299 67 67 67 109 109 109 F F F
84542024-02-08T06:42:36.557Z 15 300-319 73 73 73 117 117 117 F F F
84552024-02-08T06:42:36.558Z 16 320-339 74 74 74 118 118 118 F F F
84562024-02-08T06:42:36.558Z 17 340-359 75 75 75 119 119 119 F F F
84572024-02-08T06:42:36.558Z 18 360-379 76 76 76 120 120 120 F F F
84582024-02-08T06:42:36.558Z 19 380-399 75 75 75 119 119 119 F F F
84592024-02-08T06:42:36.558Z 20 400-419 75 75 75 119 119 119 F F F
84602024-02-08T06:42:36.558Z 21 420-439 75 75 75 119 119 119 F F F
84612024-02-08T06:42:36.558Z 22 440-459 76 76 76 120 120 120 F F F
84622024-02-08T06:42:36.558Z 23 460-479 75 75 75 119 119 119 F F F
84632024-02-08T06:42:36.558Z 24 480-499 76 76 76 120 120 120 F F F
84642024-02-08T06:42:36.558Z 25 500-519 76 76 76 120 120 120 F F F
84652024-02-08T06:42:36.558Z 26 520-539 76 76 76 120 120 120 F F F
84662024-02-08T06:42:36.558Z 27 540-559 75 75 75 119 119 119 F F F
84672024-02-08T06:42:36.558Z 28 560-579 73 73 73 117 117 117 F F F
84682024-02-08T06:42:36.558Z 29 580-599 74 74 74 118 118 118 F F F
84692024-02-08T06:42:36.558ZMax gen: 76, Max flush: 120
84702024-02-08T06:42:36.559Z
84712024-02-08T06:42:36.559ZBegin loop 39
84722024-02-08T06:42:36.559ZDownstairs to restart: 1
84732024-02-08T06:42:36.573Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.810818367Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450}
84742024-02-08T06:42:36.581Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
84752024-02-08T06:42:36.581Ztime":"2024-02-08T06:42:35.811166101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450}
84762024-02-08T06:42:36.581Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811220177Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450}
84772024-02-08T06:42:36.581Z{"msg":"Crucible a82b93ae-3138-42a9-b942-8f3e9e8b36bc has session id: ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811267103Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84782024-02-08T06:42:36.581Z{"msg":"Upstairs opts: Upstairs UUID: a82b93ae-3138-42a9-b942-8f3e9e8b36bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.81130859Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84792024-02-08T06:42:36.581Z{"msg":"Crucible stats registered with UUID: a82b93ae-3138-42a9-b942-8f3e9e8b36bc","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811353327Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84802024-02-08T06:42:36.581Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811372205Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450}
84812024-02-08T06:42:36.581Z{"msg":"a82b93ae-3138-42a9-b942-8f3e9e8b36bc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811397113Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84822024-02-08T06:42:36.582Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811413352Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84832024-02-08T06:42:36.582Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811455259Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84842024-02-08T06:42:36.582Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811485867Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84852024-02-08T06:42:36.582Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811515095Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84862024-02-08T06:42:36.582Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811547382Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84872024-02-08T06:42:36.582Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811597828Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84882024-02-08T06:42:36.582Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811645835Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"io task","client":"2","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84892024-02-08T06:42:36.582Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811691531Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"io task","client":"0","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84902024-02-08T06:42:36.582Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811737018Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"io task","client":"1","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84912024-02-08T06:42:36.582Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811767496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"io task","client":"1","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84922024-02-08T06:42:36.582Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811796943Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"io task","client":"0","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84932024-02-08T06:42:36.582Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.811818602Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"io task","client":"2","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84942024-02-08T06:42:36.582Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.812442135Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84952024-02-08T06:42:36.582Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.812487551Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84962024-02-08T06:42:36.583Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.812526388Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84972024-02-08T06:42:36.583Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.812556106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84982024-02-08T06:42:36.583Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.813006803Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
84992024-02-08T06:42:36.583Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.813148862Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85002024-02-08T06:42:36.583Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.813528573Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85012024-02-08T06:42:36.583Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.813582889Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85022024-02-08T06:42:36.583Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.813611477Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85032024-02-08T06:42:36.583Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.813646155Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85042024-02-08T06:42:36.583Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.813677192Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85052024-02-08T06:42:36.583Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.8136995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85062024-02-08T06:42:36.583Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.813759996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85072024-02-08T06:42:36.583Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.813801103Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85082024-02-08T06:42:36.588Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.814249839Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85092024-02-08T06:42:36.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.814777999Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85102024-02-08T06:42:36.588Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.814819276Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85112024-02-08T06:42:36.588Z{"msg":"[0]R flush_numbers[0..12]: [118, 119, 120, 120, 119, 118, 120, 120, 120, 118, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.814839705Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85122024-02-08T06:42:36.588Z{"msg":"[0]R generation[0..12]: [74, 75, 76, 76, 75, 74, 76, 76, 76, 74, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.814867733Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85132024-02-08T06:42:36.589Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.814887211Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85142024-02-08T06:42:36.589Z{"msg":"[1]R flush_numbers[0..12]: [118, 119, 120, 120, 119, 118, 120, 120, 120, 118, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.814913149Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85152024-02-08T06:42:36.589Z{"msg":"[1]R generation[0..12]: [74, 75, 76, 76, 75, 74, 76, 76, 76, 74, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.814930558Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85162024-02-08T06:42:36.589Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.814962636Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85172024-02-08T06:42:36.589Z{"msg":"[2]R flush_numbers[0..12]: [118, 119, 120, 120, 119, 118, 120, 120, 120, 118, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.814980454Z","Wait for a query_work_queue command to finish before sending IO
85182024-02-08T06:42:36.589Zhostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85192024-02-08T06:42:36.589Z{"msg":"[2]R generation[0..12]: [74, 75, 76, 76, 75, 74, 76, 76, 76, 74, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815003362Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85202024-02-08T06:42:36.589Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815029431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85212024-02-08T06:42:36.589Z{"msg":"Max found gen is 77","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815046339Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85222024-02-08T06:42:36.589Z{"msg":"Generation requested: 78 >= found:77","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815070708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85232024-02-08T06:42:36.589Z{"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815087786Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85242024-02-08T06:42:36.589Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815116284Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85252024-02-08T06:42:36.589Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815145342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85262024-02-08T06:42:36.589Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815161701Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85272024-02-08T06:42:36.589Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815187799Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85282024-02-08T06:42:36.590Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815204878Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85292024-02-08T06:42:36.590Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815221766Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85302024-02-08T06:42:36.590Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815237255Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85312024-02-08T06:42:36.590Z{"msg":"a82b93ae-3138-42a9-b942-8f3e9e8b36bc is now active with session: ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815254394Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85322024-02-08T06:42:36.590Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815269962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450,"session_id":"ee0ab4d5-f02d-4f08-ac2f-e65b49887fa2"}
85332024-02-08T06:42:36.590Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:35.815293691Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450}
85342024-02-08T06:42:36.590ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
85352024-02-08T06:42:36.590ZLoading write count information from file "/tmp/test_repair_verify.data"
85362024-02-08T06:42:36.590ZRun Repair workload
85372024-02-08T06:42:36.590Z01/30 Write block 27 len 20480 data: 9 9 8 8 7
85382024-02-08T06:42:36.590Z02/30 Read block 362 len 12288
85392024-02-08T06:42:36.590Z03/30 Read block 499 len 8192
85402024-02-08T06:42:36.590Z04/30 Read block 548 len 8192
85412024-02-08T06:42:36.590Z05/30 Read block 205 len 16384
85422024-02-08T06:42:36.590Z06/30 Write block 127 len 12288 data: 3 5 5
85432024-02-08T06:42:36.591Z07/30 Write block 513 len 4096 data: 4
85442024-02-08T06:42:36.591Z08/30 Read block 184 len 16384
85452024-02-08T06:42:36.591Z09/30 Read block 288 len 4096
85462024-02-08T06:42:36.591Z10/30 Read block 409 len 12288
85472024-02-08T06:42:36.591Z11/30 Read block 182 len 8192
85482024-02-08T06:42:36.591Z12/30 Write block 409 len 28672 data: 5 4 4 5 5 6 5
85492024-02-08T06:42:36.595Z13/30 Read block 323 len 20480
85502024-02-08T06:42:36.595Z14/30 Flush
85512024-02-08T06:42:36.595Z15/30 Write block 38 len 16384 data: 6 4 4 4
85522024-02-08T06:42:36.595Z16/30 Read block 424 len 4096
85532024-02-08T06:42:36.595Z17/30 Read block 9 len 20480
85542024-02-08T06:42:36.595Z18/30 Read block 542 len 8192
85552024-02-08T06:42:36.595Z19/30 Write block 156 len 16384 data: 5 5 6 5
85562024-02-08T06:42:36.595Z20/30 Read block 511 len 16384
85572024-02-08T06:42:36.595Z21/30 Read block 99 len 36864
85582024-02-08T06:42:36.598Z22/30 Read block 74 len 12288
85592024-02-08T06:42:36.598Z23/30 Write block 545 len 36864 data: 3 5 8 8 11 9 10 9 9
85602024-02-08T06:42:36.598Z24/30 Read block 585 len 40960
85612024-02-08T06:42:36.601Z25/30 Write block 87 len 32768 data: 6 6 7 9 11 13 12 11
85622024-02-08T06:42:36.601Z26/30 Write block 367 len 28672 data: 4 5 7 7 7 8 7
85632024-02-08T06:42:36.601Z27/30 Read block 575 len 24576
85642024-02-08T06:42:36.605Z28/30 Write block 124 len 12288 data: 2 2 2
85652024-02-08T06:42:36.605Z29/30 Read block 82 len 16384
85662024-02-08T06:42:36.605Z30/30 Read block 466 len 16384
85672024-02-08T06:42:36.605Z----------------------------------------------------------------
85682024-02-08T06:42:36.605Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
85692024-02-08T06:42:36.605ZStates: Active Active Active
85702024-02-08T06:42:36.605ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
85712024-02-08T06:42:36.605Z 1 Acked 1000 Write 5 Done Sent Done false
85722024-02-08T06:42:36.605Z 2 Acked 1001 Read 3 Done Sent Done false
85732024-02-08T06:42:36.605Z 3 Acked 1002 Read 2 Done Sent Done false
85742024-02-08T06:42:36.606Z 4 Acked 1003 Read 2 Done Sent Done false
85752024-02-08T06:42:36.606Z 5 Acked 1004 Read 4 Done Sent Done false
85762024-02-08T06:42:36.606Z 6 Acked 1005 Write 3 Done Sent Done false
85772024-02-08T06:42:36.606Z 7 Acked 1006 Write 1 Done Sent Done false
85782024-02-08T06:42:36.606Z 8 Acked 1007 Read 4 Done Sent Done false
85792024-02-08T06:42:36.606Z 9 Acked 1008 Read 1 Done Sent Done false
85802024-02-08T06:42:36.606Z 10 Acked 1009 Read 3 Done Sent Done false
85812024-02-08T06:42:36.606Z 11 Acked 1010 Read 2 Done Sent Done false
85822024-02-08T06:42:36.606Z 12 Acked 1011 Write 7 Done Sent Done false
85832024-02-08T06:42:36.606Z 13 Acked 1012 Read 5 Done Sent Done false
85842024-02-08T06:42:36.606Z 14 Acked 1013 Flush 0 Done Sent Done false
85852024-02-08T06:42:36.606Z 15 Acked 1014 Write 4 Done Sent Done false
85862024-02-08T06:42:36.606Z 16 Acked 1015 Read 1 Done Sent Done false
85872024-02-08T06:42:36.606Z 17 Acked 1016 Read 5 Done Sent Done false
85882024-02-08T06:42:36.606Z 18 Acked 1017 Read 2 Done Sent Done false
85892024-02-08T06:42:36.607Z 19 Acked 1018 Write 4 Done Sent Done false
85902024-02-08T06:42:36.607Z 20 Acked 1019 Read 4 Done Sent Done false
85912024-02-08T06:42:36.607Z 21 Acked 1020 Read 9 Done Sent Done false
85922024-02-08T06:42:36.607Z 22 Acked 1021 Read 3 Done Sent Done false
85932024-02-08T06:42:36.607Z 23 Acked 1022 Write 9 Done Sent Done false
85942024-02-08T06:42:36.607Z 24 Acked 1023 Read 10 Done Sent Done false
85952024-02-08T06:42:36.607Z 25 Acked 1024 Write 8 Done Sent Done false
85962024-02-08T06:42:36.607Z 26 Acked 1025 Write 7 Done Sent Done false
85972024-02-08T06:42:36.607Z 27 Acked 1026 Read 6 Done Sent Done false
85982024-02-08T06:42:36.607Z 28 Acked 1027 Write 3 Done Sent Done false
85992024-02-08T06:42:36.607Z 29 Acked 1028 Read 4 Done Sent Done false
86002024-02-08T06:42:36.607Z 30 Acked 1029 Read 4 Done Sent Done false
86012024-02-08T06:42:36.607Z STATES DS:0 DS:1 DS:2 TOTAL
86022024-02-08T06:42:36.607Z New 0 0 0 0
86032024-02-08T06:42:36.608Z Sent 0 30 0 30
86042024-02-08T06:42:36.608Z Done 30 0 30 60
86052024-02-08T06:42:36.608Z Skipped 0 0 0 0
86062024-02-08T06:42:36.608Z Error 0 0 0 0
86072024-02-08T06:42:36.608ZLast Flush: 1013 0 1013
86082024-02-08T06:42:36.608ZDownstairs last five completed:
86092024-02-08T06:42:36.608ZUpstairs last five completed: 30 29 28 27 26
86102024-02-08T06:42:36.608Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:35.842019034Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1450}
86112024-02-08T06:42:36.612ZWrote out file "/tmp/test_repair_verify.data"
86122024-02-08T06:42:36.612Z
86132024-02-08T06:42:36.612ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
86142024-02-08T06:42:36.629ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
86152024-02-08T06:42:36.629Z 0 000-019 74 74 74 118 118 118 F F F
86162024-02-08T06:42:36.629Z 1 020-039 78 75 78 121 119 121 T F T <---
86172024-02-08T06:42:36.629Z 2 040-059 76 76 76 120 120 120 T F T <---
86182024-02-08T06:42:36.629Z 3 060-079 76 76 76 120 120 120 F F F
86192024-02-08T06:42:36.629Z 4 080-099 75 75 75 119 119 119 T F T <---
86202024-02-08T06:42:36.629Z 5 100-119 74 74 74 118 118 118 F F F
86212024-02-08T06:42:36.629Z 6 120-139 78 76 78 121 120 121 T F T <---
86222024-02-08T06:42:36.629Z 7 140-159 76 76 76 120 120 120 T F T <---
86232024-02-08T06:42:36.633Z 8 160-179 76 76 76 120 120 120 F F F
86242024-02-08T06:42:36.633Z 9 180-199 74 74 74 118 118 118 F F F
86252024-02-08T06:42:36.633Z 10 200-219 73 73 73 117 117 117 F F F
86262024-02-08T06:42:36.633Z 11 220-239 73 73 73 117 117 117 F F F
86272024-02-08T06:42:36.633Z 12 240-259 76 76 76 120 120 120 F F F
86282024-02-08T06:42:36.633Z 13 260-279 69 69 69 111 111 111 F F F
86292024-02-08T06:42:36.633Z 14 280-299 67 67 67 109 109 109 F F F
86302024-02-08T06:42:36.633Z 15 300-319 73 73 73 117 117 117 F F F
86312024-02-08T06:42:36.633Z 16 320-339 74 74 74 118 118 118 F F F
86322024-02-08T06:42:36.633Z 17 340-359 75 75 75 119 119 119 F F F
86332024-02-08T06:42:36.633Z 18 360-379 76 76 76 120 120 120 T F T <---
86342024-02-08T06:42:36.633Z 19 380-399 75 75 75 119 119 119 F F F
86352024-02-08T06:42:36.633Z 20 400-419 78 75 78 121 119 121 F F F <---
86362024-02-08T06:42:36.633Z 21 420-439 75 75 75 119 119 119 F F F
86372024-02-08T06:42:36.634Z 22 440-459 76 76 76 120 120 120 F F F
86382024-02-08T06:42:36.634Z 23 460-479 75 75 75 119 119 119 F F F
86392024-02-08T06:42:36.634Z 24 480-499 76 76 76 120 120 120 F F F
86402024-02-08T06:42:36.634Z 25 500-519 78 76 78 121 120 121 F F F <---
86412024-02-08T06:42:36.634Z 26 520-539 76 76 76 120 120 120 F F F
86422024-02-08T06:42:36.634Z 27 540-559 75 75 75 119 119 119 T F T <---
86432024-02-08T06:42:36.634Z 28 560-579 73 73 73 117 117 117 F F F
86442024-02-08T06:42:36.634Z 29 580-599 74 74 74 118 118 118 F F F
86452024-02-08T06:42:36.634ZMax gen: 78, Max flush: 121
86462024-02-08T06:42:36.634ZError: Difference in extent metadata found!
86472024-02-08T06:42:36.634ZA Difference in extent metadata is expected here
86482024-02-08T06:42:36.634ZOn loop 39
86492024-02-08T06:42:36.634Z
86502024-02-08T06:42:36.637ZVerifying data now
86512024-02-08T06:42:41.111ZNo change after verify
86522024-02-08T06:42:41.111ZLoop: 39 Downstairs dump after verify (and repair):
86532024-02-08T06:42:41.126ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
86542024-02-08T06:42:41.126Z 0 000-019 74 74 74 118 118 118 F F F
86552024-02-08T06:42:41.127Z 1 020-039 79 79 79 122 122 122 F F F
86562024-02-08T06:42:41.127Z 2 040-059 79 79 79 122 122 122 F F F
86572024-02-08T06:42:41.127Z 3 060-079 76 76 76 120 120 120 F F F
86582024-02-08T06:42:41.127Z 4 080-099 79 79 79 122 122 122 F F F
86592024-02-08T06:42:41.127Z 5 100-119 74 74 74 118 118 118 F F F
86602024-02-08T06:42:41.127Z 6 120-139 79 79 79 122 122 122 F F F
86612024-02-08T06:42:41.127Z 7 140-159 79 79 79 122 122 122 F F F
86622024-02-08T06:42:41.127Z 8 160-179 76 76 76 120 120 120 F F F
86632024-02-08T06:42:41.127Z 9 180-199 74 74 74 118 118 118 F F F
86642024-02-08T06:42:41.127Z 10 200-219 73 73 73 117 117 117 F F F
86652024-02-08T06:42:41.127Z 11 220-239 73 73 73 117 117 117 F F F
86662024-02-08T06:42:41.127Z 12 240-259 76 76 76 120 120 120 F F F
86672024-02-08T06:42:41.130Z 13 260-279 69 69 69 111 111 111 F F F
86682024-02-08T06:42:41.131Z 14 280-299 67 67 67 109 109 109 F F F
86692024-02-08T06:42:41.131Z 15 300-319 73 73 73 117 117 117 F F F
86702024-02-08T06:42:41.131Z 16 320-339 74 74 74 118 118 118 F F F
86712024-02-08T06:42:41.131Z 17 340-359 75 75 75 119 119 119 F F F
86722024-02-08T06:42:41.131Z 18 360-379 79 79 79 122 122 122 F F F
86732024-02-08T06:42:41.131Z 19 380-399 75 75 75 119 119 119 F F F
86742024-02-08T06:42:41.131Z 20 400-419 78 78 78 121 121 121 F F F
86752024-02-08T06:42:41.131Z 21 420-439 75 75 75 119 119 119 F F F
86762024-02-08T06:42:41.131Z 22 440-459 76 76 76 120 120 120 F F F
86772024-02-08T06:42:41.131Z 23 460-479 75 75 75 119 119 119 F F F
86782024-02-08T06:42:41.131Z 24 480-499 76 76 76 120 120 120 F F F
86792024-02-08T06:42:41.131Z 25 500-519 78 78 78 121 121 121 F F F
86802024-02-08T06:42:41.131Z 26 520-539 76 76 76 120 120 120 F F F
86812024-02-08T06:42:41.131Z 27 540-559 79 79 79 122 122 122 F F F
86822024-02-08T06:42:41.132Z 28 560-579 73 73 73 117 117 117 F F F
86832024-02-08T06:42:41.132Z 29 580-599 74 74 74 118 118 118 F F F
86842024-02-08T06:42:41.132ZMax gen: 79, Max flush: 122
86852024-02-08T06:42:41.132Z
86862024-02-08T06:42:41.132ZBegin loop 40
86872024-02-08T06:42:41.132ZDownstairs to restart: 1
86882024-02-08T06:42:41.147Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384084106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458}
86892024-02-08T06:42:41.152Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
86902024-02-08T06:42:41.152Z,"time":"2024-02-08T06:42:40.384362755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458}
86912024-02-08T06:42:41.152Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384414751Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458}
86922024-02-08T06:42:41.152Z{"msg":"Crucible 1cc945d8-c5e2-4a47-b32f-847bd643e1c4 has session id: 60be7b4c-0781-4bfa-af0f-935b00e95f7b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384457938Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
86932024-02-08T06:42:41.152Z{"msg":"Upstairs opts: Upstairs UUID: 1cc945d8-c5e2-4a47-b32f-847bd643e1c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384492615Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
86942024-02-08T06:42:41.153Z{"msg":"Crucible stats registered with UUID: 1cc945d8-c5e2-4a47-b32f-847bd643e1c4","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384524753Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
86952024-02-08T06:42:41.153Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384547631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458}
86962024-02-08T06:42:41.153Z{"msg":"1cc945d8-c5e2-4a47-b32f-847bd643e1c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384577069Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
86972024-02-08T06:42:41.153Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384612226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
86982024-02-08T06:42:41.153Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384659573Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
86992024-02-08T06:42:41.153Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384681841Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87002024-02-08T06:42:41.153Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384703679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87012024-02-08T06:42:41.153Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384726828Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87022024-02-08T06:42:41.153Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384761995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87032024-02-08T06:42:41.153Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384848019Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"io task","client":"0","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87042024-02-08T06:42:41.153Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.384911834Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"io task","client":"2","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87052024-02-08T06:42:41.153Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.38496774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"io task","client":"1","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87062024-02-08T06:42:41.153Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.385015506Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"io task","client":"2","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87072024-02-08T06:42:41.153Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.385054343Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"io task","client":"0","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87082024-02-08T06:42:41.153Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.38509255Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"io task","client":"1","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87092024-02-08T06:42:41.154Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.385557135Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87102024-02-08T06:42:41.154Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.385594373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87112024-02-08T06:42:41.154Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.385610521Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87122024-02-08T06:42:41.154Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.38562533Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87132024-02-08T06:42:41.154Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.386214096Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87142024-02-08T06:42:41.154Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.386380764Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87152024-02-08T06:42:41.154Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.386756715Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87162024-02-08T06:42:41.154Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.386806852Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87172024-02-08T06:42:41.154Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.38683503Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87182024-02-08T06:42:41.154Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.386883516Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87192024-02-08T06:42:41.154Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.386927723Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87202024-02-08T06:42:41.154Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.386972939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87212024-02-08T06:42:41.154Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.387013236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87222024-02-08T06:42:41.154Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.387051453Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87232024-02-08T06:42:41.159Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.387523318Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87242024-02-08T06:42:41.159Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.387979393Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87252024-02-08T06:42:41.159Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.38802349Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87262024-02-08T06:42:41.160Z{"msg":"[0]R flush_numbers[0..12]: [118, 122, 122, 120, 122, 118, 122, 122, 120, 118, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388060538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87272024-02-08T06:42:41.160Z{"msg":"[0]R generation[0..12]: [74, 79, 79, 76, 79, 74, 79, 79, 76, 74, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388086785Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87282024-02-08T06:42:41.160Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388123003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87292024-02-08T06:42:41.160Z{"msg":"[1]R flush_numbers[0..12]: [118, 122, 122, 120, 122, 118, 122, 122, 120, 118, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.38815772Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87302024-02-08T06:42:41.160Z{"msg":"[1]R generation[0..12]: [74, 79, 79, 76, 79, 74, 79, 79, 76, 74, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388194517Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87312024-02-08T06:42:41.160Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388235194Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87322024-02-08T06:42:41.160ZWait for a query_work_queue command to finish before sending IO
87332024-02-08T06:42:41.160Z{"msg":"[2]R flush_numbers[0..12]: [118, 122, 122, 120, 122, 118, 122, 122, 120, 118, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388286721Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87342024-02-08T06:42:41.160Z{"msg":"[2]R generation[0..12]: [74, 79, 79, 76, 79, 74, 79, 79, 76, 74, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388318238Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87352024-02-08T06:42:41.160Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388365655Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87362024-02-08T06:42:41.160Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388393813Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87372024-02-08T06:42:41.160Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.38843019Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87382024-02-08T06:42:41.160Z{"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388464837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87392024-02-08T06:42:41.160Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388500314Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87402024-02-08T06:42:41.160Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388529362Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87412024-02-08T06:42:41.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.38856288Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87422024-02-08T06:42:41.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388597797Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87432024-02-08T06:42:41.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388624865Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87442024-02-08T06:42:41.161Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388660322Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87452024-02-08T06:42:41.161Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
87462024-02-08T06:42:41.161Z,"time":"2024-02-08T06:42:40.38869525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87472024-02-08T06:42:41.161Z{"msg":"1cc945d8-c5e2-4a47-b32f-847bd643e1c4 is now active with session: 60be7b4c-0781-4bfa-af0f-935b00e95f7b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388731247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87482024-02-08T06:42:41.161Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388745776Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458,"session_id":"60be7b4c-0781-4bfa-af0f-935b00e95f7b"}
87492024-02-08T06:42:41.161Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:40.388765345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458}
87502024-02-08T06:42:41.161ZLoading write count information from file "/tmp/test_repair_verify.data"
87512024-02-08T06:42:41.161ZRun Repair workload
87522024-02-08T06:42:41.161Z01/30 Write block 320 len 32768 data: 12 10 11 10 7 8 6 5
87532024-02-08T06:42:41.161Z02/30 Write block 79 len 4096 data: 5
87542024-02-08T06:42:41.161Z03/30 Read block 129 len 8192
87552024-02-08T06:42:41.161Z04/30 Read block 199 len 28672
87562024-02-08T06:42:41.162Z05/30 Write block 259 len 12288 data: 7 8 10
87572024-02-08T06:42:41.162Z06/30 Write block 343 len 20480 data: 7 7 8 8 6
87582024-02-08T06:42:41.162Z07/30 Write block 105 len 4096 data: 6
87592024-02-08T06:42:41.162Z08/30 Read block 149 len 40960
87602024-02-08T06:42:41.165Z09/30 Read block 305 len 36864
87612024-02-08T06:42:41.165Z10/30 Read block 90 len 20480
87622024-02-08T06:42:41.165Z11/30 Read block 473 len 36864
87632024-02-08T06:42:41.165Z12/30 Flush
87642024-02-08T06:42:41.165Z13/30 Write block 512 len 28672 data: 4 5 4 4 6 6 7
87652024-02-08T06:42:41.168Z14/30 Read block 242 len 8192
87662024-02-08T06:42:41.169Z15/30 Read block 367 len 4096
87672024-02-08T06:42:41.169Z16/30 Write block 218 len 40960 data: 6 9 10 11 11 11 13 12 11 10
87682024-02-08T06:42:41.169Z17/30 Read block 249 len 32768
87692024-02-08T06:42:41.174Z18/30 Read block 209 len 24576
87702024-02-08T06:42:41.174Z19/30 Read block 539 len 36864
87712024-02-08T06:42:41.175Z20/30 Read block 96 len 36864
87722024-02-08T06:42:41.178Z21/30 Read block 298 len 40960
87732024-02-08T06:42:41.178Z22/30 Read block 96 len 8192
87742024-02-08T06:42:41.178Z23/30 Read block 192 len 20480
87752024-02-08T06:42:41.178Z24/30 Write block 154 len 32768 data: 6 6 6 6 7 6 6 6
87762024-02-08T06:42:41.178Z25/30 Read block 428 len 4096
87772024-02-08T06:42:41.178Z26/30 Read block 94 len 36864
87782024-02-08T06:42:41.182Z27/30 Write block 390 len 12288 data: 5 6 6
87792024-02-08T06:42:41.182Z28/30 Write block 80 len 4096 data: 6
87802024-02-08T06:42:41.182Z29/30 Read block 118 len 24576
87812024-02-08T06:42:41.182Z30/30 Write block 517 len 24576 data: 7 8 9 7 5 5
87822024-02-08T06:42:41.182Z----------------------------------------------------------------
87832024-02-08T06:42:41.182Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
87842024-02-08T06:42:41.182ZStates: Active Active Active
87852024-02-08T06:42:41.182ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
87862024-02-08T06:42:41.182Z 1 Acked 1000 Write 8 Done Done Done false
87872024-02-08T06:42:41.182Z 2 Acked 1001 Write 1 Done Done Done false
87882024-02-08T06:42:41.182Z 3 Acked 1002 Read 2 Done Done Done false
87892024-02-08T06:42:41.182Z 4 Acked 1003 Read 7 Done Sent Done false
87902024-02-08T06:42:41.182Z 5 Acked 1004 Write 3 Done Sent Done false
87912024-02-08T06:42:41.183Z 6 Acked 1005 Write 5 Done Sent Done false
87922024-02-08T06:42:41.183Z 7 Acked 1006 Write 1 Done Sent Done false
87932024-02-08T06:42:41.183Z 8 Acked 1007 Read 10 Done Sent Done false
87942024-02-08T06:42:41.183Z 9 Acked 1008 Read 9 Done Sent Done false
87952024-02-08T06:42:41.183Z 10 Acked 1009 Read 5 Done Sent Done false
87962024-02-08T06:42:41.183Z 11 Acked 1010 Read 9 Done Sent Done false
87972024-02-08T06:42:41.183Z 12 Acked 1011 Flush 0 Done Sent Done false
87982024-02-08T06:42:41.183Z 13 Acked 1012 Write 7 Done Sent Done false
87992024-02-08T06:42:41.183Z 14 Acked 1013 Read 2 Done Sent Done false
88002024-02-08T06:42:41.183Z 15 Acked 1014 Read 1 Done Sent Done false
88012024-02-08T06:42:41.183Z 16 Acked 1015 Write 10 Done Sent Done false
88022024-02-08T06:42:41.183Z 17 Acked 1016 Read 8 Done Sent Done false
88032024-02-08T06:42:41.183Z 18 Acked 1017 Read 6 Done Sent Done false
88042024-02-08T06:42:41.183Z 19 Acked 1018 Read 9 Done Sent Done false
88052024-02-08T06:42:41.183Z 20 Acked 1019 Read 9 Done Sent Done false
88062024-02-08T06:42:41.183Z 21 Acked 1020 Read 10 Done Sent Done false
88072024-02-08T06:42:41.184Z 22 Acked 1021 Read 2 Done Sent Done false
88082024-02-08T06:42:41.184Z 23 Acked 1022 Read 5 Done Sent Done false
88092024-02-08T06:42:41.184Z 24 Acked 1023 Write 8 Done Sent Done false
88102024-02-08T06:42:41.184Z 25 Acked 1024 Read 1 Done Sent Done false
88112024-02-08T06:42:41.184Z 26 Acked 1025 Read 9 Done Sent Done false
88122024-02-08T06:42:41.184Z 27 Acked 1026 Write 3 Done Sent Done false
88132024-02-08T06:42:41.184Z 28 Acked 1027 Write 1 Done Sent Done false
88142024-02-08T06:42:41.184Z 29 Acked 1028 Read 6 Done Sent Done false
88152024-02-08T06:42:41.184Z 30 Acked 1029 Write 6 Sent Sent Sent false
88162024-02-08T06:42:41.184Z STATES DS:0 DS:1 DS:2 TOTAL
88172024-02-08T06:42:41.184Z New 0 0 0 0
88182024-02-08T06:42:41.184Z Sent 1 27 1 29
88192024-02-08T06:42:41.184Z Done 29 3 29 61
88202024-02-08T06:42:41.184Z Skipped 0 0 0 0
88212024-02-08T06:42:41.184Z Error 0 0 0 0
88222024-02-08T06:42:41.184ZLast Flush: 1011 0 1011
88232024-02-08T06:42:41.185ZDownstairs last five completed:
88242024-02-08T06:42:41.185ZUpstairs last five completed: 30 29 28 27 26
88252024-02-08T06:42:41.185Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:40.417805834Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1458}
88262024-02-08T06:42:41.185ZWrote out file "/tmp/test_repair_verify.data"
88272024-02-08T06:42:41.188Z
88282024-02-08T06:42:41.188ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
88292024-02-08T06:42:41.206ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88302024-02-08T06:42:41.206Z 0 000-019 74 74 74 118 118 118 F F F
88312024-02-08T06:42:41.206Z 1 020-039 79 79 79 122 122 122 F F F
88322024-02-08T06:42:41.206Z 2 040-059 79 79 79 122 122 122 F F F
88332024-02-08T06:42:41.206Z 3 060-079 80 76 80 123 120 123 F T F <---
88342024-02-08T06:42:41.206Z 4 080-099 79 79 79 122 122 122 T F T <---
88352024-02-08T06:42:41.206Z 5 100-119 80 74 80 123 118 123 F F F <---
88362024-02-08T06:42:41.206Z 6 120-139 79 79 79 122 122 122 F F F
88372024-02-08T06:42:41.209Z 7 140-159 79 79 79 122 122 122 T F T <---
88382024-02-08T06:42:41.209Z 8 160-179 76 76 76 120 120 120 T F T <---
88392024-02-08T06:42:41.209Z 9 180-199 74 74 74 118 118 118 F F F
88402024-02-08T06:42:41.209Z 10 200-219 73 73 73 117 117 117 T F T <---
88412024-02-08T06:42:41.210Z 11 220-239 73 73 73 117 117 117 T F T <---
88422024-02-08T06:42:41.210Z 12 240-259 80 76 80 123 120 123 F F F <---
88432024-02-08T06:42:41.210Z 13 260-279 80 69 80 123 111 123 F F F <---
88442024-02-08T06:42:41.210Z 14 280-299 67 67 67 109 109 109 F F F
88452024-02-08T06:42:41.210Z 15 300-319 73 73 73 117 117 117 F F F
88462024-02-08T06:42:41.210Z 16 320-339 80 74 80 123 118 123 F T F <---
88472024-02-08T06:42:41.210Z 17 340-359 80 75 80 123 119 123 F F F <---
88482024-02-08T06:42:41.210Z 18 360-379 79 79 79 122 122 122 F F F
88492024-02-08T06:42:41.210Z 19 380-399 75 75 75 119 119 119 T F T <---
88502024-02-08T06:42:41.210Z 20 400-419 78 78 78 121 121 121 F F F
88512024-02-08T06:42:41.210Z 21 420-439 75 75 75 119 119 119 F F F
88522024-02-08T06:42:41.210Z 22 440-459 76 76 76 120 120 120 F F F
88532024-02-08T06:42:41.210Z 23 460-479 75 75 75 119 119 119 F F F
88542024-02-08T06:42:41.210Z 24 480-499 76 76 76 120 120 120 F F F
88552024-02-08T06:42:41.210Z 25 500-519 78 78 78 121 121 121 T F T <---
88562024-02-08T06:42:41.211Z 26 520-539 76 76 76 120 120 120 T F T <---
88572024-02-08T06:42:41.211Z 27 540-559 79 79 79 122 122 122 F F F
88582024-02-08T06:42:41.211Z 28 560-579 73 73 73 117 117 117 F F F
88592024-02-08T06:42:41.211Z 29 580-599 74 74 74 118 118 118 F F F
88602024-02-08T06:42:41.211ZMax gen: 80, Max flush: 123
88612024-02-08T06:42:41.211ZError: Difference in extent metadata found!
88622024-02-08T06:42:41.211ZA Difference in extent metadata is expected here
88632024-02-08T06:42:41.211ZOn loop 40
88642024-02-08T06:42:41.211Z
88652024-02-08T06:42:41.214ZVerifying data now
88662024-02-08T06:42:45.883ZNo change after verify
88672024-02-08T06:42:45.887ZLoop: 40 Downstairs dump after verify (and repair):
88682024-02-08T06:42:45.901ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88692024-02-08T06:42:45.901Z 0 000-019 74 74 74 118 118 118 F F F
88702024-02-08T06:42:45.901Z 1 020-039 79 79 79 122 122 122 F F F
88712024-02-08T06:42:45.901Z 2 040-059 79 79 79 122 122 122 F F F
88722024-02-08T06:42:45.901Z 3 060-079 80 80 80 123 123 123 F F F
88732024-02-08T06:42:45.901Z 4 080-099 81 81 81 124 124 124 F F F
88742024-02-08T06:42:45.901Z 5 100-119 80 80 80 123 123 123 F F F
88752024-02-08T06:42:45.907Z 6 120-139 79 79 79 122 122 122 F F F
88762024-02-08T06:42:45.907Z 7 140-159 81 81 81 124 124 124 F F F
88772024-02-08T06:42:45.907Z 8 160-179 81 81 81 124 124 124 F F F
88782024-02-08T06:42:45.907Z 9 180-199 74 74 74 118 118 118 F F F
88792024-02-08T06:42:45.907Z 10 200-219 81 81 81 124 124 124 F F F
88802024-02-08T06:42:45.907Z 11 220-239 81 81 81 124 124 124 F F F
88812024-02-08T06:42:45.907Z 12 240-259 80 80 80 123 123 123 F F F
88822024-02-08T06:42:45.907Z 13 260-279 80 80 80 123 123 123 F F F
88832024-02-08T06:42:45.907Z 14 280-299 67 67 67 109 109 109 F F F
88842024-02-08T06:42:45.907Z 15 300-319 73 73 73 117 117 117 F F F
88852024-02-08T06:42:45.907Z 16 320-339 80 80 80 123 123 123 F F F
88862024-02-08T06:42:45.908Z 17 340-359 80 80 80 123 123 123 F F F
88872024-02-08T06:42:45.908Z 18 360-379 79 79 79 122 122 122 F F F
88882024-02-08T06:42:45.908Z 19 380-399 81 81 81 124 124 124 F F F
88892024-02-08T06:42:45.908Z 20 400-419 78 78 78 121 121 121 F F F
88902024-02-08T06:42:45.908Z 21 420-439 75 75 75 119 119 119 F F F
88912024-02-08T06:42:45.908Z 22 440-459 76 76 76 120 120 120 F F F
88922024-02-08T06:42:45.908Z 23 460-479 75 75 75 119 119 119 F F F
88932024-02-08T06:42:45.908Z 24 480-499 76 76 76 120 120 120 F F F
88942024-02-08T06:42:45.908Z 25 500-519 81 81 81 124 124 124 F F F
88952024-02-08T06:42:45.908Z 26 520-539 81 81 81 124 124 124 F F F
88962024-02-08T06:42:45.908Z 27 540-559 79 79 79 122 122 122 F F F
88972024-02-08T06:42:45.908Z 28 560-579 73 73 73 117 117 117 F F F
88982024-02-08T06:42:45.908Z 29 580-599 74 74 74 118 118 118 F F F
88992024-02-08T06:42:45.908ZMax gen: 81, Max flush: 124
89002024-02-08T06:42:45.908Z
89012024-02-08T06:42:45.908ZBegin loop 41
89022024-02-08T06:42:45.909ZDownstairs to restart: 1
89032024-02-08T06:42:45.923Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.157547584Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466}
89042024-02-08T06:42:45.923Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.157788876Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466}
89052024-02-08T06:42:45.923Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.157822674Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466Crucible runtime is spawned
89062024-02-08T06:42:45.923Z}
89072024-02-08T06:42:45.923Z{"msg":"Crucible 5ad1d190-6015-488e-9902-93ac48040f5d has session id: 05463cb9-9077-4e20-90ce-a06bad4551cf","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.157857311Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89082024-02-08T06:42:45.923Z{"msg":"Upstairs opts: Upstairs UUID: 5ad1d190-6015-488e-9902-93ac48040f5d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.157897128Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89092024-02-08T06:42:45.923Z{"msg":"Crucible stats registered with UUID: 5ad1d190-6015-488e-9902-93ac48040f5d","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.157922266Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89102024-02-08T06:42:45.923Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.157954074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466}
89112024-02-08T06:42:45.923Z{"msg":"5ad1d190-6015-488e-9902-93ac48040f5d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.1580093Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89122024-02-08T06:42:45.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158060836Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89132024-02-08T06:42:45.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158103743Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89142024-02-08T06:42:45.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.15813991Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89152024-02-08T06:42:45.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158183227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89162024-02-08T06:42:45.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158210835Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89172024-02-08T06:42:45.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158238443Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89182024-02-08T06:42:45.924Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158262261Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"io task","client":"2","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89192024-02-08T06:42:45.924Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158296148Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"io task","client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89202024-02-08T06:42:45.924Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158320186Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"io task","client":"0","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89212024-02-08T06:42:45.924Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158353064Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"io task","client":"0","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89222024-02-08T06:42:45.924Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:45.158384392Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"io task","client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89232024-02-08T06:42:45.924Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:45.158423219Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"io task","client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89242024-02-08T06:42:45.924Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:45.158482934Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89252024-02-08T06:42:45.929Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158521001Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89262024-02-08T06:42:45.929Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158551669Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89272024-02-08T06:42:45.929Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158568088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"io task","client":"2","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89282024-02-08T06:42:45.929Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.158592346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"io task","client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89292024-02-08T06:42:45.929Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.159090239Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89302024-02-08T06:42:45.930Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.159128776Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89312024-02-08T06:42:45.930Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.159155344Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89322024-02-08T06:42:45.930Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.159182862Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89332024-02-08T06:42:45.930Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.159727331Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89342024-02-08T06:42:45.930Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.159767008Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89352024-02-08T06:42:45.930Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.160324796Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89362024-02-08T06:42:45.930Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.160361923Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89372024-02-08T06:42:45.930Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.160378272Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89382024-02-08T06:42:45.930Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.1603995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89392024-02-08T06:42:45.930Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.160414279Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89402024-02-08T06:42:45.930Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:45.160434968Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89412024-02-08T06:42:46.924Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.15989937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"io task","client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89422024-02-08T06:42:46.925Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.160070487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"io task","client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89432024-02-08T06:42:46.932Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.160859678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89442024-02-08T06:42:46.932Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.160899465Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89452024-02-08T06:42:46.932Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.161396678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89462024-02-08T06:42:46.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.161904599Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89472024-02-08T06:42:46.932Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.161945996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89482024-02-08T06:42:46.932Z{"msg":"[0]R flush_numbers[0..12]: [118, 122, 122, 123, 124, 123, 122, 124, 124, 118, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.161963415Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89492024-02-08T06:42:46.932Z{"msg":"[0]R generation[0..12]: [74, 79, 79, 80, 81, 80, 79, 81, 81, 74, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.161986284Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89502024-02-08T06:42:46.932Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162001852Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89512024-02-08T06:42:46.932Z{"msg":"[1]R flush_numbers[0..12]: [118, 122, 122, 123, 124, 123, 122, 124, 124, 118, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162023531Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89522024-02-08T06:42:46.932Z{"msg":"[1]R generation[0..12]: [74, 79, 79, 80, 81, 80, 79, 81, 81, 74, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.1620381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89532024-02-08T06:42:46.932Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162053108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89542024-02-08T06:42:46.933Z{"msg":"[2]R flush_numbers[0..12]: [118, 122, 122, 123, 124, 123, 122, 124, 124, 118, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162081056Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
89552024-02-08T06:42:46.933Z05463cb9-9077-4e20-90ce-a06bad4551cf"}
89562024-02-08T06:42:46.933Z{"msg":"[2]R generation[0..12]: [74, 79, 79, 80, 81, 80, 79, 81, 81, 74, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162106095Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89572024-02-08T06:42:46.933Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162129723Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89582024-02-08T06:42:46.933Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162144292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89592024-02-08T06:42:46.933Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162185049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89602024-02-08T06:42:46.933Z{"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162206627Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89612024-02-08T06:42:46.933Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162239844Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89622024-02-08T06:42:46.933Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162271812Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89632024-02-08T06:42:46.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.16230566Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89642024-02-08T06:42:46.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162338677Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89652024-02-08T06:42:46.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162370655Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89662024-02-08T06:42:46.933Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162405342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89672024-02-08T06:42:46.934ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
89682024-02-08T06:42:46.934Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162443509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89692024-02-08T06:42:46.934Z{"msg":"5ad1d190-6015-488e-9902-93ac48040f5d is now active with session: 05463cb9-9077-4e20-90ce-a06bad4551cf","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162477996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89702024-02-08T06:42:46.934Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162500065Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466,"session_id":"05463cb9-9077-4e20-90ce-a06bad4551cf"}
89712024-02-08T06:42:46.934Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:46.162513724Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466}
89722024-02-08T06:42:46.934ZLoading write count information from file "/tmp/test_repair_verify.data"
89732024-02-08T06:42:46.934ZRun Repair workload
89742024-02-08T06:42:46.934Z01/30 Write block 310 len 20480 data: 4 4 4 5 6
89752024-02-08T06:42:46.934Z02/30 Read block 300 len 8192
89762024-02-08T06:42:46.938Z03/30 Read block 467 len 36864
89772024-02-08T06:42:46.938Z04/30 Read block 188 len 16384
89782024-02-08T06:42:46.938Z05/30 Read block 314 len 40960
89792024-02-08T06:42:46.938Z06/30 Read block 220 len 32768
89802024-02-08T06:42:46.938Z07/30 Read block 146 len 8192
89812024-02-08T06:42:46.938Z08/30 Read block 25 len 36864
89822024-02-08T06:42:46.938Z09/30 Write block 285 len 4096 data: 4
89832024-02-08T06:42:46.938Z10/30 Read block 41 len 36864
89842024-02-08T06:42:46.938Z11/30 Read block 466 len 12288
89852024-02-08T06:42:46.938Z12/30 Read block 491 len 4096
89862024-02-08T06:42:46.938Z13/30 Read block 257 len 40960
89872024-02-08T06:42:46.942Z14/30 Write block 312 len 8192 data: 5 6
89882024-02-08T06:42:46.942Z15/30 Write block 97 len 40960 data: 9 7 8 10 10 8 9 9 7 7
89892024-02-08T06:42:46.942Z16/30 Read block 62 len 32768
89902024-02-08T06:42:46.942Z17/30 Read block 500 len 20480
89912024-02-08T06:42:46.948Z18/30 Write block 432 len 16384 data: 5 4 7 7
89922024-02-08T06:42:46.948Z19/30 Read block 237 len 16384
89932024-02-08T06:42:46.948Z20/30 Write block 136 len 4096 data: 6
89942024-02-08T06:42:46.949Z21/30 Read block 302 len 32768
89952024-02-08T06:42:46.949Z22/30 Read block 576 len 8192
89962024-02-08T06:42:46.952Z23/30 Write block 247 len 36864 data: 7 8 11 10 10 7 6 7 9
89972024-02-08T06:42:46.952Z24/30 Write block 323 len 16384 data: 11 8 9 7
89982024-02-08T06:42:46.952Z25/30 Read block 307 len 20480
89992024-02-08T06:42:46.952Z26/30 Flush
90002024-02-08T06:42:46.952Z27/30 Write block 277 len 12288 data: 7 3 4
90012024-02-08T06:42:46.952Z28/30 Write block 151 len 36864 data: 6 5 5 7 7 7 7 8 7
90022024-02-08T06:42:46.952Z29/30 Read block 162 len 8192
90032024-02-08T06:42:46.956Z30/30 Read block 83 len 32768
90042024-02-08T06:42:46.956Z----------------------------------------------------------------
90052024-02-08T06:42:46.956Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
90062024-02-08T06:42:46.956ZStates: Active Active Active
90072024-02-08T06:42:46.956ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
90082024-02-08T06:42:46.956Z 1 Acked 1000 Write 5 Done Done Done false
90092024-02-08T06:42:46.956Z 2 Acked 1001 Read 2 Done Done Done false
90102024-02-08T06:42:46.956Z 3 Acked 1002 Read 9 Done Done Done false
90112024-02-08T06:42:46.957Z 4 Acked 1003 Read 4 Done Done Done false
90122024-02-08T06:42:46.957Z 5 Acked 1004 Read 10 Done Done Done false
90132024-02-08T06:42:46.957Z 6 Acked 1005 Read 8 Done Done Done false
90142024-02-08T06:42:46.957Z 7 Acked 1006 Read 2 Done Done Done false
90152024-02-08T06:42:46.957Z 8 Acked 1007 Read 9 Done Done Done false
90162024-02-08T06:42:46.957Z 9 Acked 1008 Write 1 Done Sent Done false
90172024-02-08T06:42:46.957Z 10 Acked 1009 Read 9 Done Sent Done false
90182024-02-08T06:42:46.957Z 11 Acked 1010 Read 3 Done Sent Done false
90192024-02-08T06:42:46.957Z 12 Acked 1011 Read 1 Done Sent Done false
90202024-02-08T06:42:46.957Z 13 Acked 1012 Read 10 Done Sent Done false
90212024-02-08T06:42:46.957Z 14 Acked 1013 Write 2 Done Sent Done false
90222024-02-08T06:42:46.957Z 15 Acked 1014 Write 10 Done Sent Done false
90232024-02-08T06:42:46.957Z 16 Acked 1015 Read 8 Done Sent Done false
90242024-02-08T06:42:46.957Z 17 Acked 1016 Read 5 Done Sent Done false
90252024-02-08T06:42:46.957Z 18 Acked 1017 Write 4 Done Sent Done false
90262024-02-08T06:42:46.958Z 19 Acked 1018 Read 4 Done Sent Done false
90272024-02-08T06:42:46.958Z 20 Acked 1019 Write 1 Done Sent Done false
90282024-02-08T06:42:46.958Z 21 Acked 1020 Read 8 Done Sent Done false
90292024-02-08T06:42:46.958Z 22 Acked 1021 Read 2 Done Sent Done false
90302024-02-08T06:42:46.958Z 23 Acked 1022 Write 9 Done Sent Done false
90312024-02-08T06:42:46.958Z 24 Acked 1023 Write 4 Done Sent Done false
90322024-02-08T06:42:46.958Z 25 Acked 1024 Read 5 Done Sent Done false
90332024-02-08T06:42:46.958Z 26 Acked 1025 Flush 0 Done Sent Done false
90342024-02-08T06:42:46.958Z 27 Acked 1026 Write 3 Done Sent Done false
90352024-02-08T06:42:46.958Z 28 Acked 1027 Write 9 Done Sent Done false
90362024-02-08T06:42:46.958Z 29 Acked 1028 Read 2 Done Sent Done false
90372024-02-08T06:42:46.958Z 30 Acked 1029 Read 8 Sent Sent Done false
90382024-02-08T06:42:46.958Z STATES DS:0 DS:1 DS:2 TOTAL
90392024-02-08T06:42:46.958Z New 0 0 0 0
90402024-02-08T06:42:46.958Z Sent 1 22 0 23
90412024-02-08T06:42:46.958Z Done 29 8 30 67
90422024-02-08T06:42:46.959Z Skipped 0 0 0 0
90432024-02-08T06:42:46.959Z Error 0 0 0 0
90442024-02-08T06:42:46.959ZLast Flush: 1025 0 1025
90452024-02-08T06:42:46.959ZDownstairs last five completed:
90462024-02-08T06:42:46.959ZUpstairs last five completed: 30 29 28 27 26
90472024-02-08T06:42:46.959Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:46.190855586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1466}
90482024-02-08T06:42:46.959ZWrote out file "/tmp/test_repair_verify.data"
90492024-02-08T06:42:46.962Z
90502024-02-08T06:42:46.962ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
90512024-02-08T06:42:46.979ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90522024-02-08T06:42:46.979Z 0 000-019 74 74 74 118 118 118 F F F
90532024-02-08T06:42:46.979Z 1 020-039 79 79 79 122 122 122 F F F
90542024-02-08T06:42:46.979Z 2 040-059 79 79 79 122 122 122 F F F
90552024-02-08T06:42:46.979Z 3 060-079 80 80 80 123 123 123 F F F
90562024-02-08T06:42:46.979Z 4 080-099 82 81 82 125 124 125 F F F <---
90572024-02-08T06:42:46.979Z 5 100-119 82 80 82 125 123 125 F F F <---
90582024-02-08T06:42:46.979Z 6 120-139 82 79 82 125 122 125 F F F <---
90592024-02-08T06:42:46.979Z 7 140-159 81 81 81 124 124 124 T F T <---
90602024-02-08T06:42:46.979Z 8 160-179 81 81 81 124 124 124 F F F
90612024-02-08T06:42:46.979Z 9 180-199 74 74 74 118 118 118 F F F
90622024-02-08T06:42:46.980Z 10 200-219 81 81 81 124 124 124 F F F
90632024-02-08T06:42:46.983Z 11 220-239 81 81 81 124 124 124 F F F
90642024-02-08T06:42:46.983Z 12 240-259 82 80 82 125 123 125 F F F <---
90652024-02-08T06:42:46.983Z 13 260-279 80 80 80 123 123 123 T F T <---
90662024-02-08T06:42:46.983Z 14 280-299 82 67 82 125 109 125 F F F <---
90672024-02-08T06:42:46.983Z 15 300-319 82 73 82 125 117 125 F T F <---
90682024-02-08T06:42:46.983Z 16 320-339 82 80 82 125 123 125 F F F <---
90692024-02-08T06:42:46.983Z 17 340-359 80 80 80 123 123 123 F F F
90702024-02-08T06:42:46.983Z 18 360-379 79 79 79 122 122 122 F F F
90712024-02-08T06:42:46.983Z 19 380-399 81 81 81 124 124 124 F F F
90722024-02-08T06:42:46.983Z 20 400-419 78 78 78 121 121 121 F F F
90732024-02-08T06:42:46.983Z 21 420-439 82 75 82 125 119 125 F F F <---
90742024-02-08T06:42:46.984Z 22 440-459 76 76 76 120 120 120 F F F
90752024-02-08T06:42:46.984Z 23 460-479 75 75 75 119 119 119 F F F
90762024-02-08T06:42:46.984Z 24 480-499 76 76 76 120 120 120 F F F
90772024-02-08T06:42:46.984Z 25 500-519 81 81 81 124 124 124 F F F
90782024-02-08T06:42:46.984Z 26 520-539 81 81 81 124 124 124 F F F
90792024-02-08T06:42:46.984Z 27 540-559 79 79 79 122 122 122 F F F
90802024-02-08T06:42:46.984Z 28 560-579 73 73 73 117 117 117 F F F
90812024-02-08T06:42:46.984Z 29 580-599 74 74 74 118 118 118 F F F
90822024-02-08T06:42:46.984ZMax gen: 82, Max flush: 125
90832024-02-08T06:42:46.984ZError: Difference in extent metadata found!
90842024-02-08T06:42:46.984ZA Difference in extent metadata is expected here
90852024-02-08T06:42:46.984ZOn loop 41
90862024-02-08T06:42:46.984Z
90872024-02-08T06:42:46.987ZVerifying data now
90882024-02-08T06:42:52.485ZNo change after verify
90892024-02-08T06:42:52.485ZLoop: 41 Downstairs dump after verify (and repair):
90902024-02-08T06:42:52.500ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90912024-02-08T06:42:52.500Z 0 000-019 74 74 74 118 118 118 F F F
90922024-02-08T06:42:52.506Z 1 020-039 79 79 79 122 122 122 F F F
90932024-02-08T06:42:52.506Z 2 040-059 79 79 79 122 122 122 F F F
90942024-02-08T06:42:52.507Z 3 060-079 80 80 80 123 123 123 F F F
90952024-02-08T06:42:52.507Z 4 080-099 82 82 82 125 125 125 F F F
90962024-02-08T06:42:52.507Z 5 100-119 82 82 82 125 125 125 F F F
90972024-02-08T06:42:52.507Z 6 120-139 82 82 82 125 125 125 F F F
90982024-02-08T06:42:52.507Z 7 140-159 83 83 83 126 126 126 F F F
90992024-02-08T06:42:52.507Z 8 160-179 81 81 81 124 124 124 F F F
91002024-02-08T06:42:52.507Z 9 180-199 74 74 74 118 118 118 F F F
91012024-02-08T06:42:52.507Z 10 200-219 81 81 81 124 124 124 F F F
91022024-02-08T06:42:52.507Z 11 220-239 81 81 81 124 124 124 F F F
91032024-02-08T06:42:52.507Z 12 240-259 82 82 82 125 125 125 F F F
91042024-02-08T06:42:52.507Z 13 260-279 83 83 83 126 126 126 F F F
91052024-02-08T06:42:52.507Z 14 280-299 82 82 82 125 125 125 F F F
91062024-02-08T06:42:52.507Z 15 300-319 82 82 82 125 125 125 F F F
91072024-02-08T06:42:52.507Z 16 320-339 82 82 82 125 125 125 F F F
91082024-02-08T06:42:52.508Z 17 340-359 80 80 80 123 123 123 F F F
91092024-02-08T06:42:52.508Z 18 360-379 79 79 79 122 122 122 F F F
91102024-02-08T06:42:52.508Z 19 380-399 81 81 81 124 124 124 F F F
91112024-02-08T06:42:52.508Z 20 400-419 78 78 78 121 121 121 F F F
91122024-02-08T06:42:52.508Z 21 420-439 82 82 82 125 125 125 F F F
91132024-02-08T06:42:52.508Z 22 440-459 76 76 76 120 120 120 F F F
91142024-02-08T06:42:52.508Z 23 460-479 75 75 75 119 119 119 F F F
91152024-02-08T06:42:52.508Z 24 480-499 76 76 76 120 120 120 F F F
91162024-02-08T06:42:52.508Z 25 500-519 81 81 81 124 124 124 F F F
91172024-02-08T06:42:52.508Z 26 520-539 81 81 81 124 124 124 F F F
91182024-02-08T06:42:52.508Z 27 540-559 79 79 79 122 122 122 F F F
91192024-02-08T06:42:52.508Z 28 560-579 73 73 73 117 117 117 F F F
91202024-02-08T06:42:52.508Z 29 580-599 74 74 74 118 118 118 F F F
91212024-02-08T06:42:52.508ZMax gen: 83, Max flush: 126
91222024-02-08T06:42:52.508Z
91232024-02-08T06:42:52.509ZBegin loop 42
91242024-02-08T06:42:52.509ZDownstairs to restart: 1
91252024-02-08T06:42:52.522Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.757855608Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474}
91262024-02-08T06:42:52.522Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758151346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474}
91272024-02-08T06:42:52.522Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758194893Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474}
91282024-02-08T06:42:52.527Z{"msg":"Crucible 5f11f50d-f794-41bf-9eae-f5a66663c599 has session id: bec738fb-10c7-4b2e-a813-23f45255bb16","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758222811Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91292024-02-08T06:42:52.528Z{"msg":"Upstairs opts: Upstairs UUID: 5f11f50d-f794-41bf-9eae-f5a66663c599, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758259398Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91302024-02-08T06:42:52.528Z{"msg":"Crucible stats registered with UUID: 5f11f50d-f794-41bf-9eae-f5a66663c599","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758291495Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91312024-02-08T06:42:52.528ZCrucible runtime is spawned
91322024-02-08T06:42:52.528Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758438594Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474}
91332024-02-08T06:42:52.528Z{"msg":"5f11f50d-f794-41bf-9eae-f5a66663c599 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.75849462Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91342024-02-08T06:42:52.528Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758534137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91352024-02-08T06:42:52.528Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758568225Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91362024-02-08T06:42:52.528Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758619081Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91372024-02-08T06:42:52.528Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758661958Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91382024-02-08T06:42:52.528Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758686036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91392024-02-08T06:42:52.528Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758742932Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91402024-02-08T06:42:52.528Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.75876975Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"io task","client":"0","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91412024-02-08T06:42:52.528Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758795708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"io task","client":"1","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91422024-02-08T06:42:52.528Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758828825Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"io task","client":"2","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91432024-02-08T06:42:52.529Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758862263Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"io task","client":"0","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91442024-02-08T06:42:52.529Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.75888831Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"io task","client":"1","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91452024-02-08T06:42:52.529Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.758921468Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"io task","client":"2","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91462024-02-08T06:42:52.529Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.75956044Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91472024-02-08T06:42:52.529Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.759607527Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91482024-02-08T06:42:52.529Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.759632345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91492024-02-08T06:42:52.529Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.759718628Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91502024-02-08T06:42:52.529Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.760188093Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91512024-02-08T06:42:52.529Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.760276116Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91522024-02-08T06:42:52.529Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.760817946Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91532024-02-08T06:42:52.529Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.760856713Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91542024-02-08T06:42:52.529Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.760883361Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91552024-02-08T06:42:52.529Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.760914848Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91562024-02-08T06:42:52.529Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.760940147Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91572024-02-08T06:42:52.529Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.760966714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91582024-02-08T06:42:52.529Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.76116643Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91592024-02-08T06:42:52.530Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.761202997Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91602024-02-08T06:42:52.535Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.761652873Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91612024-02-08T06:42:52.535Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762093Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91622024-02-08T06:42:52.535Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762127037Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91632024-02-08T06:42:52.535Z{"msg":"[0]R flush_numbers[0..12]: [118, 122, 122, 123, 125, 125, 125, 126, 124, 118, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762142916Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91642024-02-08T06:42:52.535Z{"msg":"[0]R generation[0..12]: [74, 79, 79, 80, 82, 82, 82, 83, 81, 74, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762164664Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91652024-02-08T06:42:52.535Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762179623Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91662024-02-08T06:42:52.535Z{"msg":"[1]R flush_numbers[0..12]: [118, 122, 122, 123, 125, 125, 125, 126, 124, 118, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762201592Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91672024-02-08T06:42:52.535Z{"msg":"[1]R generation[0..12]: [74, 79, 79, 80, 82, 82, 82, 83, 81, 74, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762216111Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91682024-02-08T06:42:52.535Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762236519Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91692024-02-08T06:42:52.535Z{"msg":"[2]R flush_numbers[0..12]: [118, 122, 122, 123, 125, 125, 125, 126, 124, 118, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762250728Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91702024-02-08T06:42:52.536ZWait for a query_work_queue command to finish before sending IO
91712024-02-08T06:42:52.536Z{"msg":"[2]R generation[0..12]: [74, 79, 79, 80, 82, 82, 82, 83, 81, 74, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762272887Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91722024-02-08T06:42:52.536Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762287145Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91732024-02-08T06:42:52.536Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762300904Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91742024-02-08T06:42:52.536Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762314423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91752024-02-08T06:42:52.536Z{"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762334102Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91762024-02-08T06:42:52.536Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762347511Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91772024-02-08T06:42:52.536Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.76236088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91782024-02-08T06:42:52.536Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762372989Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91792024-02-08T06:42:52.536Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762393837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91802024-02-08T06:42:52.536Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762407436Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91812024-02-08T06:42:52.536Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762421165Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91822024-02-08T06:42:52.536Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762434284Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91832024-02-08T06:42:52.536Z{"msg":"5f11f50d-f794-41bf-9eae-f5a66663c599 is now active with session: bec738fb-10c7-4b2e-a813-23f45255bb16","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762463412Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91842024-02-08T06:42:52.536Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.76248572Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474,"session_id":"bec738fb-10c7-4b2e-a813-23f45255bb16"}
91852024-02-08T06:42:52.537Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:51.762507649Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474}
91862024-02-08T06:42:52.537ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
91872024-02-08T06:42:52.537ZLoading write count information from file "/tmp/test_repair_verify.data"
91882024-02-08T06:42:52.537ZRun Repair workload
91892024-02-08T06:42:52.537Z01/30 Write block 227 len 28672 data: 11 12 13 10 7 7 7
91902024-02-08T06:42:52.537Z02/30 Flush
91912024-02-08T06:42:52.537Z03/30 Write block 372 len 32768 data: 9 8 3 5 6 6 5 5
91922024-02-08T06:42:52.537Z04/30 Write block 560 len 12288 data: 6 5 6
91932024-02-08T06:42:52.537Z05/30 Write block 94 len 28672 data: 12 11 10 10 8 9 11
91942024-02-08T06:42:52.541Z06/30 Write block 24 len 32768 data: 10 11 10 10 10 9 9 8
91952024-02-08T06:42:52.541Z07/30 Read block 493 len 20480
91962024-02-08T06:42:52.541Z08/30 Read block 117 len 8192
91972024-02-08T06:42:52.541Z09/30 Read block 358 len 32768
91982024-02-08T06:42:52.541Z10/30 Flush
91992024-02-08T06:42:52.541Z11/30 Write block 98 len 8192 data: 9 10
92002024-02-08T06:42:52.541Z12/30 Read block 153 len 28672
92012024-02-08T06:42:52.544Z13/30 Write block 185 len 16384 data: 8 8 7 7
92022024-02-08T06:42:52.544Z14/30 Read block 157 len 32768
92032024-02-08T06:42:52.544Z15/30 Read block 459 len 40960
92042024-02-08T06:42:52.547Z16/30 Write block 122 len 20480 data: 2 2 3 3 3
92052024-02-08T06:42:52.548Z17/30 Read block 49 len 36864
92062024-02-08T06:42:52.548Z18/30 Read block 366 len 36864
92072024-02-08T06:42:52.548Z19/30 Read block 564 len 20480
92082024-02-08T06:42:52.551Z20/30 Read block 120 len 24576
92092024-02-08T06:42:52.551Z21/30 Write block 301 len 20480 data: 10 10 9 8 8
92102024-02-08T06:42:52.551Z22/30 Write block 170 len 4096 data: 9
92112024-02-08T06:42:52.551Z23/30 Write block 559 len 20480 data: 5 7 6 7 6
92122024-02-08T06:42:52.551Z24/30 Write block 124 len 8192 data: 4 4
92132024-02-08T06:42:52.551Z25/30 Flush
92142024-02-08T06:42:52.554Z26/30 Write block 573 len 28672 data: 6 5 5 6 6 6 9
92152024-02-08T06:42:52.554Z27/30 Write block 458 len 16384 data: 9 9 5 6
92162024-02-08T06:42:52.558Z28/30 Write block 364 len 20480 data: 6 6 4 5 6
92172024-02-08T06:42:52.558Z29/30 Write block 258 len 20480 data: 6 8 9 11 9
92182024-02-08T06:42:52.558Z30/30 Read block 55 len 20480
92192024-02-08T06:42:52.558Z----------------------------------------------------------------
92202024-02-08T06:42:52.558Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:28
92212024-02-08T06:42:52.558ZStates: Active Active Active
92222024-02-08T06:42:52.558ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
92232024-02-08T06:42:52.558Z 3 Acked 1002 Write 8 Done Done Done false
92242024-02-08T06:42:52.559Z 4 Acked 1003 Write 3 Done Done Done false
92252024-02-08T06:42:52.559Z 5 Acked 1004 Write 7 Done Sent Done false
92262024-02-08T06:42:52.559Z 6 Acked 1005 Write 8 Done Sent Done false
92272024-02-08T06:42:52.559Z 7 Acked 1006 Read 5 Done Sent Done false
92282024-02-08T06:42:52.559Z 8 Acked 1007 Read 2 Done Sent Done false
92292024-02-08T06:42:52.559Z 9 Acked 1008 Read 8 Done Sent Done false
92302024-02-08T06:42:52.559Z 10 Acked 1009 Flush 0 Done Sent Done false
92312024-02-08T06:42:52.559Z 11 Acked 1010 Write 2 Done Sent Done false
92322024-02-08T06:42:52.559Z 12 Acked 1011 Read 7 Done Sent Done false
92332024-02-08T06:42:52.559Z 13 Acked 1012 Write 4 Done Sent Done false
92342024-02-08T06:42:52.559Z 14 Acked 1013 Read 8 Done Sent Done false
92352024-02-08T06:42:52.559Z 15 Acked 1014 Read 10 Done Sent Done false
92362024-02-08T06:42:52.559Z 16 Acked 1015 Write 5 Done Sent Done false
92372024-02-08T06:42:52.559Z 17 Acked 1016 Read 9 Done Sent Done false
92382024-02-08T06:42:52.559Z 18 Acked 1017 Read 9 Done Sent Done false
92392024-02-08T06:42:52.559Z 19 Acked 1018 Read 5 Done Sent Done false
92402024-02-08T06:42:52.560Z 20 Acked 1019 Read 6 Done Sent Done false
92412024-02-08T06:42:52.560Z 21 Acked 1020 Write 5 Done Sent Done false
92422024-02-08T06:42:52.560Z 22 Acked 1021 Write 1 Done Sent Done false
92432024-02-08T06:42:52.560Z 23 Acked 1022 Write 5 Done Sent Done false
92442024-02-08T06:42:52.560Z 24 Acked 1023 Write 2 Done Sent Done false
92452024-02-08T06:42:52.560Z 25 Acked 1024 Flush 0 Done Sent Done false
92462024-02-08T06:42:52.560Z 26 Acked 1025 Write 7 Done Sent Done false
92472024-02-08T06:42:52.560Z 27 Acked 1026 Write 4 Done Sent Done false
92482024-02-08T06:42:52.560Z 28 Acked 1027 Write 5 Done Sent Done false
92492024-02-08T06:42:52.560Z 29 Acked 1028 Write 5 Done Sent Done false
92502024-02-08T06:42:52.560Z 30 Acked 1029 Read 5 Done Sent Done false
92512024-02-08T06:42:52.560Z STATES DS:0 DS:1 DS:2 TOTAL
92522024-02-08T06:42:52.560Z New 0 0 0 0
92532024-02-08T06:42:52.560Z Sent 0 26 0 26
92542024-02-08T06:42:52.560Z Done 28 2 28 58
92552024-02-08T06:42:52.561Z Skipped 0 0 0 0
92562024-02-08T06:42:52.561Z Error 0 0 0 0
92572024-02-08T06:42:52.561ZLast Flush: 1024 1001 1024
92582024-02-08T06:42:52.561ZDownstairs last five completed: 1001 1000
92592024-02-08T06:42:52.561ZUpstairs last five completed: 30 29 28 27 26
92602024-02-08T06:42:52.561Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:51.793481893Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1474}
92612024-02-08T06:42:52.564ZWrote out file "/tmp/test_repair_verify.data"
92622024-02-08T06:42:52.564Z
92632024-02-08T06:42:52.567ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
92642024-02-08T06:42:52.582ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92652024-02-08T06:42:52.582Z 0 000-019 74 74 74 118 118 118 F F F
92662024-02-08T06:42:52.582Z 1 020-039 84 79 84 128 122 128 F F F <---
92672024-02-08T06:42:52.582Z 2 040-059 79 79 79 122 122 122 F F F
92682024-02-08T06:42:52.583Z 3 060-079 80 80 80 123 123 123 F F F
92692024-02-08T06:42:52.583Z 4 080-099 84 82 84 129 125 129 F F F <---
92702024-02-08T06:42:52.583Z 5 100-119 84 82 84 128 125 128 F F F <---
92712024-02-08T06:42:52.583Z 6 120-139 84 82 84 129 125 129 F F F <---
92722024-02-08T06:42:52.583Z 7 140-159 83 83 83 126 126 126 F F F
92732024-02-08T06:42:52.583Z 8 160-179 84 81 84 129 124 129 F F F <---
92742024-02-08T06:42:52.583Z 9 180-199 84 74 84 129 118 129 F F F <---
92752024-02-08T06:42:52.583Z 10 200-219 81 81 81 124 124 124 F F F
92762024-02-08T06:42:52.583Z 11 220-239 84 84 84 127 127 127 F F F
92772024-02-08T06:42:52.583Z 12 240-259 82 82 82 125 125 125 T F T <---
92782024-02-08T06:42:52.586Z 13 260-279 83 83 83 126 126 126 T F T <---
92792024-02-08T06:42:52.586Z 14 280-299 82 82 82 125 125 125 F F F
92802024-02-08T06:42:52.587Z 15 300-319 84 82 84 129 125 129 F F F <---
92812024-02-08T06:42:52.587Z 16 320-339 82 82 82 125 125 125 F F F
92822024-02-08T06:42:52.587Z 17 340-359 80 80 80 123 123 123 F F F
92832024-02-08T06:42:52.587Z 18 360-379 84 79 84 128 122 128 T T T <---
92842024-02-08T06:42:52.587Z 19 380-399 81 81 81 124 124 124 F F F
92852024-02-08T06:42:52.587Z 20 400-419 78 78 78 121 121 121 F F F
92862024-02-08T06:42:52.587Z 21 420-439 82 82 82 125 125 125 F F F
92872024-02-08T06:42:52.587Z 22 440-459 76 76 76 120 120 120 T F T <---
92882024-02-08T06:42:52.587Z 23 460-479 75 75 75 119 119 119 T F T <---
92892024-02-08T06:42:52.587Z 24 480-499 76 76 76 120 120 120 F F F
92902024-02-08T06:42:52.587Z 25 500-519 81 81 81 124 124 124 F F F
92912024-02-08T06:42:52.587Z 26 520-539 81 81 81 124 124 124 F F F
92922024-02-08T06:42:52.587Z 27 540-559 84 79 84 129 122 129 F F F <---
92932024-02-08T06:42:52.587Z 28 560-579 84 73 84 129 117 129 T T T <---
92942024-02-08T06:42:52.588Z 29 580-599 74 74 74 118 118 118 F F F
92952024-02-08T06:42:52.588ZMax gen: 84, Max flush: 129
92962024-02-08T06:42:52.588ZError: Difference in extent metadata found!
92972024-02-08T06:42:52.588ZA Difference in extent metadata is expected here
92982024-02-08T06:42:52.588ZOn loop 42
92992024-02-08T06:42:52.588Z
93002024-02-08T06:42:52.591ZVerifying data now
93012024-02-08T06:42:58.266ZNo change after verify
93022024-02-08T06:42:58.266ZLoop: 42 Downstairs dump after verify (and repair):
93032024-02-08T06:42:58.281ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93042024-02-08T06:42:58.281Z 0 000-019 74 74 74 118 118 118 F F F
93052024-02-08T06:42:58.281Z 1 020-039 84 84 84 128 128 128 F F F
93062024-02-08T06:42:58.281Z 2 040-059 79 79 79 122 122 122 F F F
93072024-02-08T06:42:58.284Z 3 060-079 80 80 80 123 123 123 F F F
93082024-02-08T06:42:58.285Z 4 080-099 84 84 84 129 129 129 F F F
93092024-02-08T06:42:58.285Z 5 100-119 84 84 84 128 128 128 F F F
93102024-02-08T06:42:58.285Z 6 120-139 84 84 84 129 129 129 F F F
93112024-02-08T06:42:58.285Z 7 140-159 83 83 83 126 126 126 F F F
93122024-02-08T06:42:58.285Z 8 160-179 84 84 84 129 129 129 F F F
93132024-02-08T06:42:58.285Z 9 180-199 84 84 84 129 129 129 F F F
93142024-02-08T06:42:58.285Z 10 200-219 81 81 81 124 124 124 F F F
93152024-02-08T06:42:58.285Z 11 220-239 84 84 84 127 127 127 F F F
93162024-02-08T06:42:58.285Z 12 240-259 85 85 85 130 130 130 F F F
93172024-02-08T06:42:58.285Z 13 260-279 85 85 85 130 130 130 F F F
93182024-02-08T06:42:58.285Z 14 280-299 82 82 82 125 125 125 F F F
93192024-02-08T06:42:58.285Z 15 300-319 84 84 84 129 129 129 F F F
93202024-02-08T06:42:58.285Z 16 320-339 82 82 82 125 125 125 F F F
93212024-02-08T06:42:58.285Z 17 340-359 80 80 80 123 123 123 F F F
93222024-02-08T06:42:58.286Z 18 360-379 85 85 85 130 130 130 F F F
93232024-02-08T06:42:58.286Z 19 380-399 81 81 81 124 124 124 F F F
93242024-02-08T06:42:58.286Z 20 400-419 78 78 78 121 121 121 F F F
93252024-02-08T06:42:58.286Z 21 420-439 82 82 82 125 125 125 F F F
93262024-02-08T06:42:58.286Z 22 440-459 85 85 85 130 130 130 F F F
93272024-02-08T06:42:58.286Z 23 460-479 85 85 85 130 130 130 F F F
93282024-02-08T06:42:58.286Z 24 480-499 76 76 76 120 120 120 F F F
93292024-02-08T06:42:58.286Z 25 500-519 81 81 81 124 124 124 F F F
93302024-02-08T06:42:58.286Z 26 520-539 81 81 81 124 124 124 F F F
93312024-02-08T06:42:58.286Z 27 540-559 84 84 84 129 129 129 F F F
93322024-02-08T06:42:58.286Z 28 560-579 85 85 85 130 130 130 F F F
93332024-02-08T06:42:58.286Z 29 580-599 74 74 74 118 118 118 F F F
93342024-02-08T06:42:58.286ZMax gen: 85, Max flush: 130
93352024-02-08T06:42:58.286Z
93362024-02-08T06:42:58.286ZBegin loop 43
93372024-02-08T06:42:58.287ZDownstairs to restart: 2
93382024-02-08T06:42:58.302Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537060727Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482}
93392024-02-08T06:42:58.302Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\nCrucible runtime is spawned
93402024-02-08T06:42:58.307Z rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537392743Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482}
93412024-02-08T06:42:58.307Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537433629Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482}
93422024-02-08T06:42:58.308Z{"msg":"Crucible 18869e74-d315-4f9b-8b20-4ae358e3bf92 has session id: 34d38151-282c-4e2c-8c74-da20f547dfc1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537459098Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93432024-02-08T06:42:58.308Z{"msg":"Upstairs opts: Upstairs UUID: 18869e74-d315-4f9b-8b20-4ae358e3bf92, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537495755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93442024-02-08T06:42:58.308Z{"msg":"Crucible stats registered with UUID: 18869e74-d315-4f9b-8b20-4ae358e3bf92","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537528882Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93452024-02-08T06:42:58.308Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537551851Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482}
93462024-02-08T06:42:58.308Z{"msg":"18869e74-d315-4f9b-8b20-4ae358e3bf92 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537577059Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93472024-02-08T06:42:58.308Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537599247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93482024-02-08T06:42:58.308Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537633314Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93492024-02-08T06:42:58.308Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537659543Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93502024-02-08T06:42:58.308Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537700469Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93512024-02-08T06:42:58.308Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537725618Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93522024-02-08T06:42:58.308Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537759515Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93532024-02-08T06:42:58.308Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537784683Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93542024-02-08T06:42:58.308Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.5378249Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"io task","client":"0","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93552024-02-08T06:42:58.308Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537851918Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"io task","client":"1","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93562024-02-08T06:42:58.308Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:57.537893035Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93572024-02-08T06:42:58.309Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:57.537923423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93582024-02-08T06:42:58.309Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:57.53795329Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93592024-02-08T06:42:58.309Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.537986708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93602024-02-08T06:42:58.309Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.538010196Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93612024-02-08T06:42:58.309Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.538044863Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93622024-02-08T06:42:58.309Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.538073991Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"io task","client":"1","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93632024-02-08T06:42:58.309Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.538111089Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"io task","client":"0","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93642024-02-08T06:42:58.309Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.538643549Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93652024-02-08T06:42:58.309Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.538693415Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93662024-02-08T06:42:58.309Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.538724552Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93672024-02-08T06:42:58.309Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.538763169Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93682024-02-08T06:42:58.309Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.539261692Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93692024-02-08T06:42:58.309Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.539371424Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93702024-02-08T06:42:58.309Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.539743646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93712024-02-08T06:42:58.310Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.539802862Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93722024-02-08T06:42:58.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.539832079Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93732024-02-08T06:42:58.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.539851488Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93742024-02-08T06:42:58.310Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.539871556Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93752024-02-08T06:42:58.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:57.539886255Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93762024-02-08T06:42:59.414Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.539646775Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93772024-02-08T06:42:59.422Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.539864439Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93782024-02-08T06:42:59.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.540846085Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93792024-02-08T06:42:59.422Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.540883572Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93802024-02-08T06:42:59.422Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.541386654Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93812024-02-08T06:42:59.422Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.541900346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93822024-02-08T06:42:59.422Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.541950802Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93832024-02-08T06:42:59.422Z{"msg":"[0]R flush_numbers[0..12]: [118, 128, 122, 123, 129, 128, 129, 126, 129, 129, 124, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.54197167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93842024-02-08T06:42:59.423Z{"msg":"[0]R generation[0..12]: [74, 84, 79, 80, 84, 84, 84, 83, 84, 84, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542000008Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93852024-02-08T06:42:59.423Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542018597Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93862024-02-08T06:42:59.423Z{"msg":"[1]R flush_numbers[0..12]: [118, 128, 122, 123, 129, 128, 129, 126, 129, 129, 124, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542046055Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93872024-02-08T06:42:59.423Z{"msg":"[1]R generation[0..12]: [74, 84, 79, 80, 84, 84, 84, 83, 84, 84, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542063834Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93882024-02-08T06:42:59.423Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542090222Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93892024-02-08T06:42:59.423Z{"msg":"[2]R flush_numbers[0..12]: [118, 128, 122, 123, 129, 128, 129, 126, 129, 129, 124, 127]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
93902024-02-08T06:42:59.423Z,"time":"2024-02-08T06:42:58.54210872Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93912024-02-08T06:42:59.423Z{"msg":"[2]R generation[0..12]: [74, 84, 79, 80, 84, 84, 84, 83, 84, 84, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542138188Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93922024-02-08T06:42:59.423Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542156487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93932024-02-08T06:42:59.423Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542174315Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93942024-02-08T06:42:59.423Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542201073Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93952024-02-08T06:42:59.423Z{"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542217922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93962024-02-08T06:42:59.423Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.5422428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93972024-02-08T06:42:59.423Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542260059Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93982024-02-08T06:42:59.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542292437Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
93992024-02-08T06:42:59.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542309455Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
94002024-02-08T06:42:59.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542329704Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
94012024-02-08T06:42:59.424Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542343313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
94022024-02-08T06:42:59.424Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542356202Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
94032024-02-08T06:42:59.424Z{"msg":"18869e74-d315-4f9b-8b20-4ae358e3bf92 is now active with session: 34d38151-282c-4e2c-8c74-da20f547dfc1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542369311Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
94042024-02-08T06:42:59.424Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542388099Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482,"session_id":"34d38151-282c-4e2c-8c74-da20f547dfc1"}
94052024-02-08T06:42:59.424Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:42:58.542401008Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482}
94062024-02-08T06:42:59.424ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
94072024-02-08T06:42:59.424ZLoading write count information from file "/tmp/test_repair_verify.data"
94082024-02-08T06:42:59.424ZRun Repair workload
94092024-02-08T06:42:59.424Z01/30 Write block 96 len 20480 data: 11 11 10 11 12
94102024-02-08T06:42:59.424Z02/30 Read block 202 len 36864
94112024-02-08T06:42:59.424Z03/30 Write block 331 len 32768 data: 8 8 9 11 10 12 11 12
94122024-02-08T06:42:59.424Z04/30 Read block 500 len 4096
94132024-02-08T06:42:59.425Z05/30 Write block 388 len 8192 data: 6 7
94142024-02-08T06:42:59.425Z06/30 Read block 507 len 12288
94152024-02-08T06:42:59.425Z07/30 Flush
94162024-02-08T06:42:59.425Z08/30 Write block 289 len 8192 data: 6 8
94172024-02-08T06:42:59.425Z09/30 Write block 555 len 8192 data: 8 11
94182024-02-08T06:42:59.425Z10/30 Read block 424 len 28672
94192024-02-08T06:42:59.425Z11/30 Read block 6 len 40960
94202024-02-08T06:42:59.425Z12/30 Read block 93 len 16384
94212024-02-08T06:42:59.425Z13/30 Write block 383 len 28672 data: 5 8 7 6 6 7 8
94222024-02-08T06:42:59.425Z14/30 Flush
94232024-02-08T06:42:59.425Z15/30 Write block 94 len 32768 data: 13 12 12 12 11 12 13 11
94242024-02-08T06:42:59.425Z16/30 Write block 462 len 32768 data: 7 7 5 4 5 7 7 8
94252024-02-08T06:42:59.425Z17/30 Write block 382 len 36864 data: 6 6 9 8 7 7 8 9 6
94262024-02-08T06:42:59.425Z18/30 Write block 53 len 24576 data: 6 8 8 10 9 5
94272024-02-08T06:42:59.425Z19/30 Read block 195 len 32768
94282024-02-08T06:42:59.425Z20/30 Write block 542 len 36864 data: 5 5 5 4 6 9 9 12 10
94292024-02-08T06:42:59.426Z21/30 Read block 470 len 32768
94302024-02-08T06:42:59.426Z22/30 Read block 257 len 16384
94312024-02-08T06:42:59.426Z23/30 Flush
94322024-02-08T06:42:59.426Z24/30 Write block 364 len 40960 data: 7 7 5 6 7 8 8 8 10 9
94332024-02-08T06:42:59.426Z25/30 Read block 148 len 4096
94342024-02-08T06:42:59.426Z26/30 Read block 490 len 24576
94352024-02-08T06:42:59.426Z27/30 Write block 404 len 12288 data: 8 8 7
94362024-02-08T06:42:59.426Z28/30 Write block 556 len 8192 data: 12 8
94372024-02-08T06:42:59.426Z29/30 Read block 214 len 16384
94382024-02-08T06:42:59.426Z30/30 Write block 490 len 24576 data: 7 8 7 7 7 7
94392024-02-08T06:42:59.426Z----------------------------------------------------------------
94402024-02-08T06:42:59.426Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
94412024-02-08T06:42:59.426ZStates: Active Active Active
94422024-02-08T06:42:59.426ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
94432024-02-08T06:42:59.426Z 1 Acked 1000 Write 5 Done Done Done false
94442024-02-08T06:42:59.426Z 2 Acked 1001 Read 9 Done Done Done false
94452024-02-08T06:42:59.426Z 3 Acked 1002 Write 8 Done Done Done false
94462024-02-08T06:42:59.426Z 4 Acked 1003 Read 1 Done Done Done false
94472024-02-08T06:42:59.427Z 5 Acked 1004 Write 2 Done Done Done false
94482024-02-08T06:42:59.427Z 6 Acked 1005 Read 3 Done Done Sent false
94492024-02-08T06:42:59.427Z 7 Acked 1006 Flush 0 Done Done Sent false
94502024-02-08T06:42:59.427Z 8 Acked 1007 Write 2 Done Done Sent false
94512024-02-08T06:42:59.427Z 9 Acked 1008 Write 2 Done Done Sent false
94522024-02-08T06:42:59.427Z 10 Acked 1009 Read 7 Done Done Sent false
94532024-02-08T06:42:59.427Z 11 Acked 1010 Read 10 Done Done Sent false
94542024-02-08T06:42:59.427Z 12 Acked 1011 Read 4 Done Done Sent false
94552024-02-08T06:42:59.427Z 13 Acked 1012 Write 7 Done Done Sent false
94562024-02-08T06:42:59.427Z 14 Acked 1013 Flush 0 Done Done Sent false
94572024-02-08T06:42:59.427Z 15 Acked 1014 Write 8 Done Done Sent false
94582024-02-08T06:42:59.427Z 16 Acked 1015 Write 8 Done Done Sent false
94592024-02-08T06:42:59.427Z 17 Acked 1016 Write 9 Done Done Sent false
94602024-02-08T06:42:59.427Z 18 Acked 1017 Write 6 Done Done Sent false
94612024-02-08T06:42:59.427Z 19 Acked 1018 Read 8 Done Done Sent false
94622024-02-08T06:42:59.427Z 20 Acked 1019 Write 9 Done Done Sent false
94632024-02-08T06:42:59.427Z 21 Acked 1020 Read 8 Done Done Sent false
94642024-02-08T06:42:59.427Z 22 Acked 1021 Read 4 Done Done Sent false
94652024-02-08T06:42:59.427Z 23 Acked 1022 Flush 0 Done Done Sent false
94662024-02-08T06:42:59.428Z 24 Acked 1023 Write 10 Done Done Sent false
94672024-02-08T06:42:59.428Z 25 Acked 1024 Read 1 Done Done Sent false
94682024-02-08T06:42:59.428Z 26 Acked 1025 Read 6 Done Done Sent false
94692024-02-08T06:42:59.428Z 27 Acked 1026 Write 3 Done Done Sent false
94702024-02-08T06:42:59.428Z 28 Acked 1027 Write 2 Done Done Sent false
94712024-02-08T06:42:59.428Z 29 Acked 1028 Read 4 Done Done Sent false
94722024-02-08T06:42:59.428Z 30 Acked 1029 Write 6 Sent Sent Sent false
94732024-02-08T06:42:59.428Z STATES DS:0 DS:1 DS:2 TOTAL
94742024-02-08T06:42:59.428Z New 0 0 0 0
94752024-02-08T06:42:59.428Z Sent 1 1 25 27
94762024-02-08T06:42:59.428Z Done 29 29 5 63
94772024-02-08T06:42:59.433Z Skipped 0 0 0 0
94782024-02-08T06:42:59.433Z Error 0 0 0 0
94792024-02-08T06:42:59.433ZLast Flush: 1022 1022 0
94802024-02-08T06:42:59.446ZDownstairs last five completed:
94812024-02-08T06:42:59.446ZUpstairs last five completed: 30 29 28 27 26
94822024-02-08T06:42:59.446Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:42:58.573126971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1482}
94832024-02-08T06:42:59.446ZWrote out file "/tmp/test_repair_verify.data"
94842024-02-08T06:42:59.446Z
94852024-02-08T06:42:59.446ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
94862024-02-08T06:42:59.446ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94872024-02-08T06:42:59.446Z 0 000-019 74 74 74 118 118 118 F F F
94882024-02-08T06:42:59.447Z 1 020-039 84 84 84 128 128 128 F F F
94892024-02-08T06:42:59.447Z 2 040-059 86 86 79 133 133 122 F F F <---
94902024-02-08T06:42:59.447Z 3 060-079 80 80 80 123 123 123 F F F
94912024-02-08T06:42:59.447Z 4 080-099 86 86 84 133 133 129 F F T <---
94922024-02-08T06:42:59.447Z 5 100-119 86 86 84 133 133 128 F F T <---
94932024-02-08T06:42:59.447Z 6 120-139 84 84 84 129 129 129 F F F
94942024-02-08T06:42:59.447Z 7 140-159 83 83 83 126 126 126 F F F
94952024-02-08T06:42:59.447Z 8 160-179 84 84 84 129 129 129 F F F
94962024-02-08T06:42:59.447Z 9 180-199 84 84 84 129 129 129 F F F
94972024-02-08T06:42:59.448Z 10 200-219 81 81 81 124 124 124 F F F
94982024-02-08T06:42:59.448Z 11 220-239 84 84 84 127 127 127 F F F
94992024-02-08T06:42:59.448Z 12 240-259 85 85 85 130 130 130 F F F
95002024-02-08T06:42:59.448Z 13 260-279 85 85 85 130 130 130 F F F
95012024-02-08T06:42:59.448Z 14 280-299 86 86 82 132 132 125 F F F <---
95022024-02-08T06:42:59.448Z 15 300-319 84 84 84 129 129 129 F F F
95032024-02-08T06:42:59.448Z 16 320-339 86 86 82 131 131 125 F F T <---
95042024-02-08T06:42:59.448Z 17 340-359 80 80 80 123 123 123 F F F
95052024-02-08T06:42:59.449Z 18 360-379 85 85 85 130 130 130 T T F <---
95062024-02-08T06:42:59.449Z 19 380-399 86 86 81 133 133 124 F F T <---
95072024-02-08T06:42:59.449Z 20 400-419 78 78 78 121 121 121 T T F <---
95082024-02-08T06:42:59.449Z 21 420-439 82 82 82 125 125 125 F F F
95092024-02-08T06:42:59.449Z 22 440-459 85 85 85 130 130 130 F F F
95102024-02-08T06:42:59.449Z 23 460-479 86 86 85 133 133 130 F F F <---
95112024-02-08T06:42:59.449Z 24 480-499 76 76 76 120 120 120 T T F <---
95122024-02-08T06:42:59.449Z 25 500-519 81 81 81 124 124 124 F F F
95132024-02-08T06:42:59.449Z 26 520-539 81 81 81 124 124 124 F F F
95142024-02-08T06:42:59.450Z 27 540-559 86 86 84 133 133 129 T T F <---
95152024-02-08T06:42:59.450Z 28 560-579 85 85 85 130 130 130 F F F
95162024-02-08T06:42:59.450Z 29 580-599 74 74 74 118 118 118 F F F
95172024-02-08T06:42:59.450ZMax gen: 86, Max flush: 133
95182024-02-08T06:42:59.450ZError: Difference in extent metadata found!
95192024-02-08T06:42:59.450ZA Difference in extent metadata is expected here
95202024-02-08T06:42:59.450ZOn loop 43
95212024-02-08T06:42:59.450Z
95222024-02-08T06:42:59.451ZVerifying data now
95232024-02-08T06:43:04.924ZNo change after verify
95242024-02-08T06:43:04.924ZLoop: 43 Downstairs dump after verify (and repair):
95252024-02-08T06:43:04.937ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95262024-02-08T06:43:04.937Z 0 000-019 74 74 74 118 118 118 F F F
95272024-02-08T06:43:04.937Z 1 020-039 84 84 84 128 128 128 F F F
95282024-02-08T06:43:04.938Z 2 040-059 86 86 86 133 133 133 F F F
95292024-02-08T06:43:04.938Z 3 060-079 80 80 80 123 123 123 F F F
95302024-02-08T06:43:04.938Z 4 080-099 86 86 86 133 133 133 F F F
95312024-02-08T06:43:04.938Z 5 100-119 86 86 86 133 133 133 F F F
95322024-02-08T06:43:04.938Z 6 120-139 84 84 84 129 129 129 F F F
95332024-02-08T06:43:04.938Z 7 140-159 83 83 83 126 126 126 F F F
95342024-02-08T06:43:04.938Z 8 160-179 84 84 84 129 129 129 F F F
95352024-02-08T06:43:04.941Z 9 180-199 84 84 84 129 129 129 F F F
95362024-02-08T06:43:04.941Z 10 200-219 81 81 81 124 124 124 F F F
95372024-02-08T06:43:04.941Z 11 220-239 84 84 84 127 127 127 F F F
95382024-02-08T06:43:04.942Z 12 240-259 85 85 85 130 130 130 F F F
95392024-02-08T06:43:04.942Z 13 260-279 85 85 85 130 130 130 F F F
95402024-02-08T06:43:04.942Z 14 280-299 86 86 86 132 132 132 F F F
95412024-02-08T06:43:04.942Z 15 300-319 84 84 84 129 129 129 F F F
95422024-02-08T06:43:04.942Z 16 320-339 86 86 86 131 131 131 F F F
95432024-02-08T06:43:04.942Z 17 340-359 80 80 80 123 123 123 F F F
95442024-02-08T06:43:04.942Z 18 360-379 87 87 87 134 134 134 F F F
95452024-02-08T06:43:04.942Z 19 380-399 86 86 86 133 133 133 F F F
95462024-02-08T06:43:04.942Z 20 400-419 87 87 87 134 134 134 F F F
95472024-02-08T06:43:04.942Z 21 420-439 82 82 82 125 125 125 F F F
95482024-02-08T06:43:04.942Z 22 440-459 85 85 85 130 130 130 F F F
95492024-02-08T06:43:04.942Z 23 460-479 86 86 86 133 133 133 F F F
95502024-02-08T06:43:04.942Z 24 480-499 87 87 87 134 134 134 F F F
95512024-02-08T06:43:04.942Z 25 500-519 81 81 81 124 124 124 F F F
95522024-02-08T06:43:04.943Z 26 520-539 81 81 81 124 124 124 F F F
95532024-02-08T06:43:04.943Z 27 540-559 87 87 87 134 134 134 F F F
95542024-02-08T06:43:04.943Z 28 560-579 85 85 85 130 130 130 F F F
95552024-02-08T06:43:04.943Z 29 580-599 74 74 74 118 118 118 F F F
95562024-02-08T06:43:04.943ZMax gen: 87, Max flush: 134
95572024-02-08T06:43:04.943Z
95582024-02-08T06:43:04.943ZBegin loop 44
95592024-02-08T06:43:04.943ZDownstairs to restart: 2
95602024-02-08T06:43:04.957Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.191240276Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490}
95612024-02-08T06:43:04.963Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.191545803Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490}
95622024-02-08T06:43:04.963Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
95632024-02-08T06:43:04.963Z,"time":"2024-02-08T06:43:04.191605298Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490}
95642024-02-08T06:43:04.963Z{"msg":"Crucible d17239e7-690e-4fd5-a698-d91a62019281 has session id: 2dc70435-d453-4c2a-a072-e94fa348ce36","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.191649055Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95652024-02-08T06:43:04.963Z{"msg":"Upstairs opts: Upstairs UUID: d17239e7-690e-4fd5-a698-d91a62019281, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.191689142Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95662024-02-08T06:43:04.963Z{"msg":"Crucible stats registered with UUID: d17239e7-690e-4fd5-a698-d91a62019281","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.19171496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95672024-02-08T06:43:04.963Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.191782805Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490}
95682024-02-08T06:43:04.963Z{"msg":"d17239e7-690e-4fd5-a698-d91a62019281 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.191862459Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95692024-02-08T06:43:04.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.191911965Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95702024-02-08T06:43:04.963Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192021757Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95712024-02-08T06:43:04.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192053455Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95722024-02-08T06:43:04.964Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192130779Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95732024-02-08T06:43:04.964Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192162957Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95742024-02-08T06:43:04.964Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192198774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95752024-02-08T06:43:04.964Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192237971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95762024-02-08T06:43:04.964Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192286447Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95772024-02-08T06:43:04.964Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192310765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95782024-02-08T06:43:04.964Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192330144Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95792024-02-08T06:43:04.964Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192353722Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95802024-02-08T06:43:04.964Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192368521Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95812024-02-08T06:43:04.964Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.19290916Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95822024-02-08T06:43:04.964Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192950857Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95832024-02-08T06:43:04.964Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.192976635Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95842024-02-08T06:43:04.964Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.193012563Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95852024-02-08T06:43:04.964Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.193620537Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95862024-02-08T06:43:04.964Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.193682182Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95872024-02-08T06:43:04.965Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.194098661Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95882024-02-08T06:43:04.965Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.194145637Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95892024-02-08T06:43:04.965Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.194296336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95902024-02-08T06:43:04.965Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.194368971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95912024-02-08T06:43:04.965Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.194405458Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95922024-02-08T06:43:04.965Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.194421517Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95932024-02-08T06:43:04.965Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.194449005Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95942024-02-08T06:43:04.965Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.194466044Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95952024-02-08T06:43:04.965Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.1946383Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95962024-02-08T06:43:04.972Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195111815Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95972024-02-08T06:43:04.972Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195145962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95982024-02-08T06:43:04.972Z{"msg":"[0]R flush_numbers[0..12]: [118, 128, 133, 123, 133, 133, 129, 126, 129, 129, 124, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195163061Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
95992024-02-08T06:43:04.972Z{"msg":"[0]R generation[0..12]: [74, 84, 86, 80, 86, 86, 84, 83, 84, 84, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.19518587Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96002024-02-08T06:43:04.972Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195201868Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96012024-02-08T06:43:04.972Z{"msg":"[1]R flush_numbers[0..12]: [118, 128, 133, 123, 133, 133, 129, 126, 129, 129, 124, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195223587Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96022024-02-08T06:43:04.973Z{"msg":"[1]R generation[0..12]: [74, 84, 86, 80, 86, 86, 84, 83, 84, 84, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195238186Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96032024-02-08T06:43:04.973Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195259864Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96042024-02-08T06:43:04.973Z{"msg":"[2]R flush_numbers[0..12]: [118, 128, 133, 123, 133, 133, 129, 126, 129, 129, 124, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195275033Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96052024-02-08T06:43:04.973ZWait for a query_work_queue command to finish before sending IO
96062024-02-08T06:43:04.973Z{"msg":"[2]R generation[0..12]: [74, 84, 86, 80, 86, 86, 84, 83, 84, 84, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.19530631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96072024-02-08T06:43:04.973Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195338388Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96082024-02-08T06:43:04.973Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195362466Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96092024-02-08T06:43:04.973Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195394094Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96102024-02-08T06:43:04.973Z{"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195425332Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96112024-02-08T06:43:04.973Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195455609Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96122024-02-08T06:43:04.973Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195487747Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96132024-02-08T06:43:04.973Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195517305Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96142024-02-08T06:43:04.973Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195549602Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96152024-02-08T06:43:04.973Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195566091Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96162024-02-08T06:43:04.973Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195596669Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96172024-02-08T06:43:04.974Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195615407Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96182024-02-08T06:43:04.974Z{"msg":"d17239e7-690e-4fd5-a698-d91a62019281 is now active with session: 2dc70435-d453-4c2a-a072-e94fa348ce36","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
96192024-02-08T06:43:04.974Z,"time":"2024-02-08T06:43:04.195643005Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96202024-02-08T06:43:04.974Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.19571669Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490,"session_id":"2dc70435-d453-4c2a-a072-e94fa348ce36"}
96212024-02-08T06:43:04.974Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:04.195757116Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490}
96222024-02-08T06:43:04.974ZLoading write count information from file "/tmp/test_repair_verify.data"
96232024-02-08T06:43:04.974ZRun Repair workload
96242024-02-08T06:43:04.974Z01/30 Write block 42 len 24576 data: 5 4 4 3 5 3
96252024-02-08T06:43:04.974Z02/30 Read block 1 len 40960
96262024-02-08T06:43:04.974Z03/30 Read block 262 len 28672
96272024-02-08T06:43:04.974Z04/30 Read block 247 len 40960
96282024-02-08T06:43:04.974Z05/30 Read block 91 len 40960
96292024-02-08T06:43:04.974Z06/30 Read block 542 len 24576
96302024-02-08T06:43:04.974Z07/30 Read block 404 len 24576
96312024-02-08T06:43:04.978Z08/30 Read block 451 len 20480
96322024-02-08T06:43:04.978Z09/30 Write block 226 len 40960 data: 12 12 13 14 11 8 8 8 7 8
96332024-02-08T06:43:04.978Z10/30 Write block 131 len 8192 data: 8 8
96342024-02-08T06:43:04.978Z11/30 Flush
96352024-02-08T06:43:04.979Z12/30 Write block 134 len 8192 data: 6 5
96362024-02-08T06:43:04.979Z13/30 Read block 194 len 40960
96372024-02-08T06:43:04.979Z14/30 Flush
96382024-02-08T06:43:04.979Z15/30 Write block 325 len 28672 data: 10 8 6 5 5 5 9
96392024-02-08T06:43:04.983Z16/30 Read block 405 len 40960
96402024-02-08T06:43:04.983Z17/30 Flush
96412024-02-08T06:43:04.983Z18/30 Write block 235 len 4096 data: 9
96422024-02-08T06:43:04.983Z19/30 Flush
96432024-02-08T06:43:04.986Z20/30 Write block 156 len 28672 data: 8 8 9 8 7 7 9
96442024-02-08T06:43:04.986Z21/30 Read block 382 len 4096
96452024-02-08T06:43:04.986Z22/30 Read block 221 len 20480
96462024-02-08T06:43:04.989Z23/30 Flush
96472024-02-08T06:43:04.989Z24/30 Write block 486 len 12288 data: 9 9 9
96482024-02-08T06:43:04.993Z25/30 Write block 547 len 40960 data: 10 10 13 11 11 10 10 9 9 13
96492024-02-08T06:43:04.993Z26/30 Read block 334 len 36864
96502024-02-08T06:43:04.993Z27/30 Read block 185 len 4096
96512024-02-08T06:43:04.993Z28/30 Read block 7 len 28672
96522024-02-08T06:43:04.997Z29/30 Write block 194 len 36864 data: 10 9 8 10 11 11 10 9 9
96532024-02-08T06:43:04.997Z30/30 Write block 203 len 12288 data: 7 6 7
96542024-02-08T06:43:04.997Z----------------------------------------------------------------
96552024-02-08T06:43:04.997Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
96562024-02-08T06:43:04.998ZStates: Active Active Active
96572024-02-08T06:43:04.998ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
96582024-02-08T06:43:04.998Z 1 Acked 1000 Write 6 Done Done Done false
96592024-02-08T06:43:04.998Z 2 Acked 1001 Read 10 Done Done Done false
96602024-02-08T06:43:04.998Z 3 Acked 1002 Read 7 Done Done Done false
96612024-02-08T06:43:04.998Z 4 Acked 1003 Read 10 Done Done Done false
96622024-02-08T06:43:04.998Z 5 Acked 1004 Read 10 Done Done Done false
96632024-02-08T06:43:04.998Z 6 Acked 1005 Read 6 Done Done Done false
96642024-02-08T06:43:04.998Z 7 Acked 1006 Read 6 Done Done Done false
96652024-02-08T06:43:04.998Z 8 Acked 1007 Read 5 Done Done Sent false
96662024-02-08T06:43:04.998Z 9 Acked 1008 Write 10 Done Done Sent false
96672024-02-08T06:43:04.998Z 10 Acked 1009 Write 2 Done Done Sent false
96682024-02-08T06:43:04.998Z 11 Acked 1010 Flush 0 Done Done Sent false
96692024-02-08T06:43:04.998Z 12 Acked 1011 Write 2 Done Done Sent false
96702024-02-08T06:43:04.998Z 13 Acked 1012 Read 10 Done Done Sent false
96712024-02-08T06:43:04.999Z 14 Acked 1013 Flush 0 Done Done Sent false
96722024-02-08T06:43:04.999Z 15 Acked 1014 Write 7 Done Done Sent false
96732024-02-08T06:43:04.999Z 16 Acked 1015 Read 10 Done Done Sent false
96742024-02-08T06:43:04.999Z 17 Acked 1016 Flush 0 Done Done Sent false
96752024-02-08T06:43:04.999Z 18 Acked 1017 Write 1 Done Done Sent false
96762024-02-08T06:43:04.999Z 19 Acked 1018 Flush 0 Done Done Sent false
96772024-02-08T06:43:04.999Z 20 Acked 1019 Write 7 Done Done Sent false
96782024-02-08T06:43:04.999Z 21 Acked 1020 Read 1 Done Done Sent false
96792024-02-08T06:43:04.999Z 22 Acked 1021 Read 5 Done Done Sent false
96802024-02-08T06:43:04.999Z 23 Acked 1022 Flush 0 Done Done Sent false
96812024-02-08T06:43:04.999Z 24 Acked 1023 Write 3 Done Done Sent false
96822024-02-08T06:43:04.999Z 25 Acked 1024 Write 10 Done Done Sent false
96832024-02-08T06:43:04.999Z 26 Acked 1025 Read 9 Done Done Sent false
96842024-02-08T06:43:04.999Z 27 Acked 1026 Read 1 Done Done Sent false
96852024-02-08T06:43:04.999Z 28 Acked 1027 Read 7 Done Done Sent false
96862024-02-08T06:43:05.000Z 29 Acked 1028 Write 9 Sent Sent Sent false
96872024-02-08T06:43:05.000Z 30 Acked 1029 Write 3 Sent Sent Sent false
96882024-02-08T06:43:05.000Z STATES DS:0 DS:1 DS:2 TOTAL
96892024-02-08T06:43:05.000Z New 0 0 0 0
96902024-02-08T06:43:05.000Z Sent 2 2 23 27
96912024-02-08T06:43:05.000Z Done 28 28 7 63
96922024-02-08T06:43:05.000Z Skipped 0 0 0 0
96932024-02-08T06:43:05.000Z Error 0 0 0 0
96942024-02-08T06:43:05.000ZLast Flush: 1022 1022 0
96952024-02-08T06:43:05.000ZDownstairs last five completed:
96962024-02-08T06:43:05.000ZUpstairs last five completed: 30 29 28 27 26
96972024-02-08T06:43:05.000Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:04.230419914Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1490}
96982024-02-08T06:43:05.000ZWrote out file "/tmp/test_repair_verify.data"
96992024-02-08T06:43:05.004Z
97002024-02-08T06:43:05.004ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
97012024-02-08T06:43:05.019ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97022024-02-08T06:43:05.019Z 0 000-019 74 74 74 118 118 118 F F F
97032024-02-08T06:43:05.020Z 1 020-039 84 84 84 128 128 128 F F F
97042024-02-08T06:43:05.020Z 2 040-059 88 88 86 135 135 133 F F T <---
97052024-02-08T06:43:05.020Z 3 060-079 80 80 80 123 123 123 F F F
97062024-02-08T06:43:05.020Z 4 080-099 86 86 86 133 133 133 F F F
97072024-02-08T06:43:05.020Z 5 100-119 86 86 86 133 133 133 F F F
97082024-02-08T06:43:05.020Z 6 120-139 88 88 84 136 136 129 F F F <---
97092024-02-08T06:43:05.020Z 7 140-159 88 88 83 139 139 126 F F F <---
97102024-02-08T06:43:05.020Z 8 160-179 88 88 84 139 139 129 F F F <---
97112024-02-08T06:43:05.024Z 9 180-199 84 84 84 129 129 129 T T F <---
97122024-02-08T06:43:05.024Z 10 200-219 81 81 81 124 124 124 T T F <---
97132024-02-08T06:43:05.024Z 11 220-239 88 88 84 138 138 127 F F F <---
97142024-02-08T06:43:05.024Z 12 240-259 85 85 85 130 130 130 F F F
97152024-02-08T06:43:05.024Z 13 260-279 85 85 85 130 130 130 F F F
97162024-02-08T06:43:05.024Z 14 280-299 86 86 86 132 132 132 F F F
97172024-02-08T06:43:05.024Z 15 300-319 84 84 84 129 129 129 F F F
97182024-02-08T06:43:05.024Z 16 320-339 88 88 86 137 137 131 F F F <---
97192024-02-08T06:43:05.024Z 17 340-359 80 80 80 123 123 123 F F F
97202024-02-08T06:43:05.024Z 18 360-379 87 87 87 134 134 134 F F F
97212024-02-08T06:43:05.025Z 19 380-399 86 86 86 133 133 133 F F F
97222024-02-08T06:43:05.025Z 20 400-419 87 87 87 134 134 134 F F F
97232024-02-08T06:43:05.025Z 21 420-439 82 82 82 125 125 125 F F F
97242024-02-08T06:43:05.025Z 22 440-459 85 85 85 130 130 130 F F F
97252024-02-08T06:43:05.025Z 23 460-479 86 86 86 133 133 133 F F F
97262024-02-08T06:43:05.025Z 24 480-499 87 87 87 134 134 134 T T F <---
97272024-02-08T06:43:05.025Z 25 500-519 81 81 81 124 124 124 F F F
97282024-02-08T06:43:05.025Z 26 520-539 81 81 81 124 124 124 F F F
97292024-02-08T06:43:05.025Z 27 540-559 87 87 87 134 134 134 T T F <---
97302024-02-08T06:43:05.025Z 28 560-579 85 85 85 130 130 130 F F F
97312024-02-08T06:43:05.025Z 29 580-599 74 74 74 118 118 118 F F F
97322024-02-08T06:43:05.025ZMax gen: 88, Max flush: 139
97332024-02-08T06:43:05.025ZError: Difference in extent metadata found!
97342024-02-08T06:43:05.025ZA Difference in extent metadata is expected here
97352024-02-08T06:43:05.025ZOn loop 44
97362024-02-08T06:43:05.025Z
97372024-02-08T06:43:05.029ZVerifying data now
97382024-02-08T06:43:06.551ZNo change after verify
97392024-02-08T06:43:06.551ZLoop: 44 Downstairs dump after verify (and repair):
97402024-02-08T06:43:06.566ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97412024-02-08T06:43:06.566Z 0 000-019 74 74 74 118 118 118 F F F
97422024-02-08T06:43:06.566Z 1 020-039 84 84 84 128 128 128 F F F
97432024-02-08T06:43:06.567Z 2 040-059 88 88 88 135 135 135 F F F
97442024-02-08T06:43:06.567Z 3 060-079 80 80 80 123 123 123 F F F
97452024-02-08T06:43:06.567Z 4 080-099 86 86 86 133 133 133 F F F
97462024-02-08T06:43:06.567Z 5 100-119 86 86 86 133 133 133 F F F
97472024-02-08T06:43:06.567Z 6 120-139 88 88 88 136 136 136 F F F
97482024-02-08T06:43:06.567Z 7 140-159 88 88 88 139 139 139 F F F
97492024-02-08T06:43:06.567Z 8 160-179 88 88 88 139 139 139 F F F
97502024-02-08T06:43:06.567Z 9 180-199 89 89 89 140 140 140 F F F
97512024-02-08T06:43:06.570Z 10 200-219 89 89 89 140 140 140 F F F
97522024-02-08T06:43:06.570Z 11 220-239 88 88 88 138 138 138 F F F
97532024-02-08T06:43:06.571Z 12 240-259 85 85 85 130 130 130 F F F
97542024-02-08T06:43:06.571Z 13 260-279 85 85 85 130 130 130 F F F
97552024-02-08T06:43:06.571Z 14 280-299 86 86 86 132 132 132 F F F
97562024-02-08T06:43:06.571Z 15 300-319 84 84 84 129 129 129 F F F
97572024-02-08T06:43:06.571Z 16 320-339 88 88 88 137 137 137 F F F
97582024-02-08T06:43:06.571Z 17 340-359 80 80 80 123 123 123 F F F
97592024-02-08T06:43:06.571Z 18 360-379 87 87 87 134 134 134 F F F
97602024-02-08T06:43:06.571Z 19 380-399 86 86 86 133 133 133 F F F
97612024-02-08T06:43:06.571Z 20 400-419 87 87 87 134 134 134 F F F
97622024-02-08T06:43:06.571Z 21 420-439 82 82 82 125 125 125 F F F
97632024-02-08T06:43:06.571Z 22 440-459 85 85 85 130 130 130 F F F
97642024-02-08T06:43:06.571Z 23 460-479 86 86 86 133 133 133 F F F
97652024-02-08T06:43:06.571Z 24 480-499 89 89 89 140 140 140 F F F
97662024-02-08T06:43:06.571Z 25 500-519 81 81 81 124 124 124 F F F
97672024-02-08T06:43:06.571Z 26 520-539 81 81 81 124 124 124 F F F
97682024-02-08T06:43:06.572Z 27 540-559 89 89 89 140 140 140 F F F
97692024-02-08T06:43:06.572Z 28 560-579 85 85 85 130 130 130 F F F
97702024-02-08T06:43:06.572Z 29 580-599 74 74 74 118 118 118 F F F
97712024-02-08T06:43:06.572ZMax gen: 89, Max flush: 140
97722024-02-08T06:43:06.572Z
97732024-02-08T06:43:06.572ZBegin loop 45
97742024-02-08T06:43:06.572ZDownstairs to restart: 2
97752024-02-08T06:43:06.587Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.82095653Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498}
97762024-02-08T06:43:06.587ZCrucible runtime is spawned
97772024-02-08T06:43:06.593Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821327392Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498}
97782024-02-08T06:43:06.593Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821377848Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498}
97792024-02-08T06:43:06.593Z{"msg":"Crucible b7d9d28e-fa7b-4c67-81f0-fe64c8f4386f has session id: a6ed3a5d-d2ef-429d-ad43-20a334f482b5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821409216Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97802024-02-08T06:43:06.594Z{"msg":"Upstairs opts: Upstairs UUID: b7d9d28e-fa7b-4c67-81f0-fe64c8f4386f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821471431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97812024-02-08T06:43:06.594Z{"msg":"Crucible stats registered with UUID: b7d9d28e-fa7b-4c67-81f0-fe64c8f4386f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821515908Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97822024-02-08T06:43:06.594Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821560505Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498}
97832024-02-08T06:43:06.594Z{"msg":"b7d9d28e-fa7b-4c67-81f0-fe64c8f4386f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821588383Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97842024-02-08T06:43:06.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821639339Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97852024-02-08T06:43:06.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821693715Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97862024-02-08T06:43:06.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821800967Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97872024-02-08T06:43:06.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821879801Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97882024-02-08T06:43:06.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.821946216Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97892024-02-08T06:43:06.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822003621Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97902024-02-08T06:43:06.594Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822054068Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"io task","client":"2","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97912024-02-08T06:43:06.594Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822092635Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"io task","client":"0","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97922024-02-08T06:43:06.594Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822132272Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"io task","client":"1","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97932024-02-08T06:43:06.594Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822176778Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"io task","client":"1","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97942024-02-08T06:43:06.594Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822207006Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"io task","client":"0","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97952024-02-08T06:43:06.595Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822230424Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"io task","client":"2","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97962024-02-08T06:43:06.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822663372Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97972024-02-08T06:43:06.595Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822705109Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97982024-02-08T06:43:06.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822733197Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
97992024-02-08T06:43:06.595Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.822771404Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98002024-02-08T06:43:06.595Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.8233568Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98012024-02-08T06:43:06.595Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.823450763Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98022024-02-08T06:43:06.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.823714203Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98032024-02-08T06:43:06.595Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.823763389Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98042024-02-08T06:43:06.595Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.823900539Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98052024-02-08T06:43:06.595Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.823939406Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98062024-02-08T06:43:06.595Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.823955965Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98072024-02-08T06:43:06.595Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.823982603Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98082024-02-08T06:43:06.595Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824011101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98092024-02-08T06:43:06.595Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824040428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98102024-02-08T06:43:06.595Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824242153Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98112024-02-08T06:43:06.596Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824668931Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98122024-02-08T06:43:06.596Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824710728Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98132024-02-08T06:43:06.596Z{"msg":"[0]R flush_numbers[0..12]: [118, 128, 135, 123, 133, 133, 136, 139, 139, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824728157Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98142024-02-08T06:43:06.596Z{"msg":"[0]R generation[0..12]: [74, 84, 88, 80, 86, 86, 88, 88, 88, 89, 89, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824749635Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98152024-02-08T06:43:06.596Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824764444Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98162024-02-08T06:43:06.596Z{"msg":"[1]R flush_numbers[0..12]: [118, 128, 135, 123, 133, 133, 136, 139, 139, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824785932Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98172024-02-08T06:43:06.596Z{"msg":"[1]R generation[0..12]: [74, 84, 88, 80, 86, 86, 88, 88, 88, 89, 89, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824800101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98182024-02-08T06:43:06.596Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.82482064Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98192024-02-08T06:43:06.596Z{"msg":"[2]R flush_numbers[0..12]: [118, 128, 135, 123, 133, 133, 136, 139, 139, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824835689Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98202024-02-08T06:43:06.596ZWait for a query_work_queue command to finish before sending IO
98212024-02-08T06:43:06.596Z{"msg":"[2]R generation[0..12]: [74, 84, 88, 80, 86, 86, 88, 88, 88, 89, 89, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824857907Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98222024-02-08T06:43:06.596Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824872346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98232024-02-08T06:43:06.596Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824892624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98242024-02-08T06:43:06.596Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824906474Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98252024-02-08T06:43:06.597Z{"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824919542Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98262024-02-08T06:43:06.597Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824939511Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98272024-02-08T06:43:06.597Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.82495352Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98282024-02-08T06:43:06.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824966919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98292024-02-08T06:43:06.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.824980328Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98302024-02-08T06:43:06.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.825014555Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98312024-02-08T06:43:06.597Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.825029774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98322024-02-08T06:43:06.597Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.825049423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98332024-02-08T06:43:06.597Z{"msg":"b7d9d28e-fa7b-4c67-81f0-fe64c8f4386f is now active with session: a6ed3a5d-d2ef-429d-ad43-20a334f482b5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.825062862Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98342024-02-08T06:43:06.597Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.82508138Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498,"session_id":"a6ed3a5d-d2ef-429d-ad43-20a334f482b5"}
98352024-02-08T06:43:06.604Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:05.825094179Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498}
98362024-02-08T06:43:06.604ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
98372024-02-08T06:43:06.604ZLoading write count information from file "/tmp/test_repair_verify.data"
98382024-02-08T06:43:06.604ZRun Repair workload
98392024-02-08T06:43:06.604Z01/30 Write block 430 len 12288 data: 6 5 6
98402024-02-08T06:43:06.604Z02/30 Flush
98412024-02-08T06:43:06.604Z03/30 Flush
98422024-02-08T06:43:06.604Z04/30 Write block 376 len 4096 data: 7
98432024-02-08T06:43:06.604Z05/30 Write block 125 len 8192 data: 5 4
98442024-02-08T06:43:06.604Z06/30 Flush
98452024-02-08T06:43:06.604Z07/30 Write block 16 len 8192 data: 4 5
98462024-02-08T06:43:06.604Z08/30 Read block 472 len 16384
98472024-02-08T06:43:06.604Z09/30 Write block 138 len 8192 data: 8 7
98482024-02-08T06:43:06.605Z10/30 Write block 81 len 36864 data: 5 6 7 7 7 7 7 7 8
98492024-02-08T06:43:06.609Z11/30 Read block 72 len 16384
98502024-02-08T06:43:06.609Z12/30 Write block 318 len 4096 data: 7
98512024-02-08T06:43:06.609Z13/30 Read block 46 len 36864
98522024-02-08T06:43:06.609Z14/30 Read block 393 len 8192
98532024-02-08T06:43:06.609Z15/30 Read block 8 len 16384
98542024-02-08T06:43:06.609Z16/30 Read block 326 len 8192
98552024-02-08T06:43:06.609Z17/30 Write block 90 len 36864 data: 10 12 14 13 14 13 13 13 12
98562024-02-08T06:43:06.610Z18/30 Write block 455 len 4096 data: 9
98572024-02-08T06:43:06.610Z19/30 Flush
98582024-02-08T06:43:06.614Z20/30 Flush
98592024-02-08T06:43:06.614Z21/30 Write block 18 len 12288 data: 6 6 7
98602024-02-08T06:43:06.614Z22/30 Read block 555 len 40960
98612024-02-08T06:43:06.614Z23/30 Read block 480 len 40960
98622024-02-08T06:43:06.614Z24/30 Read block 554 len 24576
98632024-02-08T06:43:06.618Z25/30 Write block 569 len 40960 data: 5 7 6 6 7 6 6 7 7 7
98642024-02-08T06:43:06.618Z26/30 Read block 280 len 24576
98652024-02-08T06:43:06.618Z27/30 Write block 333 len 16384 data: 10 12 11 13
98662024-02-08T06:43:06.618Z28/30 Write block 544 len 8192 data: 6 5
98672024-02-08T06:43:06.618Z29/30 Write block 62 len 4096 data: 5
98682024-02-08T06:43:06.622Z30/30 Write block 404 len 28672 data: 9 9 8 7 6 6 5
98692024-02-08T06:43:06.622Z----------------------------------------------------------------
98702024-02-08T06:43:06.622Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:27
98712024-02-08T06:43:06.622ZStates: Active Active Active
98722024-02-08T06:43:06.622ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
98732024-02-08T06:43:06.623Z 4 Acked 1003 Write 1 Done Done Sent false
98742024-02-08T06:43:06.623Z 5 Acked 1004 Write 2 Done Done Sent false
98752024-02-08T06:43:06.623Z 6 Acked 1005 Flush 0 Done Done Sent false
98762024-02-08T06:43:06.623Z 7 Acked 1006 Write 2 Done Done Sent false
98772024-02-08T06:43:06.623Z 8 Acked 1007 Read 4 Done Done Sent false
98782024-02-08T06:43:06.623Z 9 Acked 1008 Write 2 Done Done Sent false
98792024-02-08T06:43:06.623Z 10 Acked 1009 Write 9 Done Done Sent false
98802024-02-08T06:43:06.623Z 11 Acked 1010 Read 4 Done Done Sent false
98812024-02-08T06:43:06.623Z 12 Acked 1011 Write 1 Done Done Sent false
98822024-02-08T06:43:06.623Z 13 Acked 1012 Read 9 Done Done Sent false
98832024-02-08T06:43:06.623Z 14 Acked 1013 Read 2 Done Done Sent false
98842024-02-08T06:43:06.623Z 15 Acked 1014 Read 4 Done Done Sent false
98852024-02-08T06:43:06.624Z 16 Acked 1015 Read 2 Done Done Sent false
98862024-02-08T06:43:06.624Z 17 Acked 1016 Write 9 Done Done Sent false
98872024-02-08T06:43:06.624Z 18 Acked 1017 Write 1 Done Done Sent false
98882024-02-08T06:43:06.624Z 19 Acked 1018 Flush 0 Done Done Sent false
98892024-02-08T06:43:06.624Z 20 Acked 1019 Flush 0 Done Done Sent false
98902024-02-08T06:43:06.624Z 21 Acked 1020 Write 3 Done Done Sent false
98912024-02-08T06:43:06.624Z 22 Acked 1021 Read 10 Done Done Sent false
98922024-02-08T06:43:06.624Z 23 Acked 1022 Read 10 Done Done Sent false
98932024-02-08T06:43:06.624Z 24 Acked 1023 Read 6 Done Done Sent false
98942024-02-08T06:43:06.624Z 25 Acked 1024 Write 10 Done Done Sent false
98952024-02-08T06:43:06.624Z 26 Acked 1025 Read 6 Done Done Sent false
98962024-02-08T06:43:06.624Z 27 Acked 1026 Write 4 Done Done Sent false
98972024-02-08T06:43:06.624Z 28 Acked 1027 Write 2 Done Done Sent false
98982024-02-08T06:43:06.624Z 29 Acked 1028 Write 1 Done Done Sent false
98992024-02-08T06:43:06.624Z 30 Acked 1029 Write 7 Sent Sent Sent false
99002024-02-08T06:43:06.625Z STATES DS:0 DS:1 DS:2 TOTAL
99012024-02-08T06:43:06.625Z New 0 0 0 0
99022024-02-08T06:43:06.625Z Sent 1 1 27 29
99032024-02-08T06:43:06.625Z Done 26 26 0 52
99042024-02-08T06:43:06.625Z Skipped 0 0 0 0
99052024-02-08T06:43:06.625Z Error 0 0 0 0
99062024-02-08T06:43:06.625ZLast Flush: 1019 1019 1002
99072024-02-08T06:43:06.625ZDownstairs last five completed: 1002 1001 1000
99082024-02-08T06:43:06.625ZUpstairs last five completed: 30 29 28 27 26
99092024-02-08T06:43:06.625Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:05.853617108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1498}
99102024-02-08T06:43:06.625ZWrote out file "/tmp/test_repair_verify.data"
99112024-02-08T06:43:06.629Z
99122024-02-08T06:43:06.629ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
99132024-02-08T06:43:06.646ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99142024-02-08T06:43:06.646Z 0 000-019 90 90 74 144 144 118 T T F <---
99152024-02-08T06:43:06.646Z 1 020-039 84 84 84 128 128 128 T T F <---
99162024-02-08T06:43:06.646Z 2 040-059 88 88 88 135 135 135 F F F
99172024-02-08T06:43:06.646Z 3 060-079 80 80 80 123 123 123 T T F <---
99182024-02-08T06:43:06.646Z 4 080-099 90 90 86 144 144 133 F F F <---
99192024-02-08T06:43:06.646Z 5 100-119 86 86 86 133 133 133 F F F
99202024-02-08T06:43:06.646Z 6 120-139 90 90 88 144 144 136 F F F <---
99212024-02-08T06:43:06.646Z 7 140-159 88 88 88 139 139 139 F F F
99222024-02-08T06:43:06.647Z 8 160-179 88 88 88 139 139 139 F F F
99232024-02-08T06:43:06.650Z 9 180-199 89 89 89 140 140 140 F F F
99242024-02-08T06:43:06.650Z 10 200-219 89 89 89 140 140 140 F F F
99252024-02-08T06:43:06.650Z 11 220-239 88 88 88 138 138 138 F F F
99262024-02-08T06:43:06.650Z 12 240-259 85 85 85 130 130 130 F F F
99272024-02-08T06:43:06.650Z 13 260-279 85 85 85 130 130 130 F F F
99282024-02-08T06:43:06.650Z 14 280-299 86 86 86 132 132 132 F F F
99292024-02-08T06:43:06.650Z 15 300-319 90 90 84 144 144 129 F F F <---
99302024-02-08T06:43:06.650Z 16 320-339 88 88 88 137 137 137 T T F <---
99312024-02-08T06:43:06.651Z 17 340-359 80 80 80 123 123 123 F F F
99322024-02-08T06:43:06.651Z 18 360-379 90 90 87 143 143 134 F F F <---
99332024-02-08T06:43:06.651Z 19 380-399 86 86 86 133 133 133 F F F
99342024-02-08T06:43:06.651Z 20 400-419 87 87 87 134 134 134 T T F <---
99352024-02-08T06:43:06.651Z 21 420-439 90 90 90 141 141 141 F F F
99362024-02-08T06:43:06.651Z 22 440-459 90 90 85 144 144 130 F F F <---
99372024-02-08T06:43:06.651Z 23 460-479 86 86 86 133 133 133 F F F
99382024-02-08T06:43:06.651Z 24 480-499 89 89 89 140 140 140 F F F
99392024-02-08T06:43:06.651Z 25 500-519 81 81 81 124 124 124 F F F
99402024-02-08T06:43:06.651Z 26 520-539 81 81 81 124 124 124 F F F
99412024-02-08T06:43:06.651Z 27 540-559 89 89 89 140 140 140 T T F <---
99422024-02-08T06:43:06.651Z 28 560-579 85 85 85 130 130 130 T T F <---
99432024-02-08T06:43:06.651Z 29 580-599 74 74 74 118 118 118 F F F
99442024-02-08T06:43:06.651ZMax gen: 90, Max flush: 144
99452024-02-08T06:43:06.651ZError: Difference in extent metadata found!
99462024-02-08T06:43:06.652ZA Difference in extent metadata is expected here
99472024-02-08T06:43:06.652ZOn loop 45
99482024-02-08T06:43:06.652Z
99492024-02-08T06:43:06.652ZVerifying data now
99502024-02-08T06:43:11.251ZNo change after verify
99512024-02-08T06:43:11.252ZLoop: 45 Downstairs dump after verify (and repair):
99522024-02-08T06:43:11.267ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99532024-02-08T06:43:11.267Z 0 000-019 91 91 91 145 145 145 F F F
99542024-02-08T06:43:11.267Z 1 020-039 91 91 91 145 145 145 F F F
99552024-02-08T06:43:11.267Z 2 040-059 88 88 88 135 135 135 F F F
99562024-02-08T06:43:11.267Z 3 060-079 91 91 91 145 145 145 F F F
99572024-02-08T06:43:11.267Z 4 080-099 90 90 90 144 144 144 F F F
99582024-02-08T06:43:11.267Z 5 100-119 86 86 86 133 133 133 F F F
99592024-02-08T06:43:11.271Z 6 120-139 90 90 90 144 144 144 F F F
99602024-02-08T06:43:11.271Z 7 140-159 88 88 88 139 139 139 F F F
99612024-02-08T06:43:11.271Z 8 160-179 88 88 88 139 139 139 F F F
99622024-02-08T06:43:11.271Z 9 180-199 89 89 89 140 140 140 F F F
99632024-02-08T06:43:11.272Z 10 200-219 89 89 89 140 140 140 F F F
99642024-02-08T06:43:11.272Z 11 220-239 88 88 88 138 138 138 F F F
99652024-02-08T06:43:11.272Z 12 240-259 85 85 85 130 130 130 F F F
99662024-02-08T06:43:11.272Z 13 260-279 85 85 85 130 130 130 F F F
99672024-02-08T06:43:11.272Z 14 280-299 86 86 86 132 132 132 F F F
99682024-02-08T06:43:11.272Z 15 300-319 90 90 90 144 144 144 F F F
99692024-02-08T06:43:11.272Z 16 320-339 91 91 91 145 145 145 F F F
99702024-02-08T06:43:11.272Z 17 340-359 80 80 80 123 123 123 F F F
99712024-02-08T06:43:11.272Z 18 360-379 90 90 90 143 143 143 F F F
99722024-02-08T06:43:11.273Z 19 380-399 86 86 86 133 133 133 F F F
99732024-02-08T06:43:11.273Z 20 400-419 91 91 91 145 145 145 F F F
99742024-02-08T06:43:11.273Z 21 420-439 90 90 90 141 141 141 F F F
99752024-02-08T06:43:11.273Z 22 440-459 90 90 90 144 144 144 F F F
99762024-02-08T06:43:11.273Z 23 460-479 86 86 86 133 133 133 F F F
99772024-02-08T06:43:11.273Z 24 480-499 89 89 89 140 140 140 F F F
99782024-02-08T06:43:11.273Z 25 500-519 81 81 81 124 124 124 F F F
99792024-02-08T06:43:11.273Z 26 520-539 81 81 81 124 124 124 F F F
99802024-02-08T06:43:11.274Z 27 540-559 91 91 91 145 145 145 F F F
99812024-02-08T06:43:11.274Z 28 560-579 91 91 91 145 145 145 F F F
99822024-02-08T06:43:11.274Z 29 580-599 74 74 74 118 118 118 F F F
99832024-02-08T06:43:11.274ZMax gen: 91, Max flush: 145
99842024-02-08T06:43:11.274Z
99852024-02-08T06:43:11.274ZBegin loop 46
99862024-02-08T06:43:11.274ZDownstairs to restart: 1
99872024-02-08T06:43:11.297Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.522297813Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506}
99882024-02-08T06:43:11.297Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\nCrucible runtime is spawned
99892024-02-08T06:43:11.304Z git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.522617639Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506}
99902024-02-08T06:43:11.304Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.522670785Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506}
99912024-02-08T06:43:11.304Z{"msg":"Crucible e7a3c4c7-153d-4d49-a9b8-a44325f536f6 has session id: 08ada59e-91eb-4b8e-92f9-7117df99d362","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.522700102Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
99922024-02-08T06:43:11.304Z{"msg":"Upstairs opts: Upstairs UUID: e7a3c4c7-153d-4d49-a9b8-a44325f536f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.522746089Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
99932024-02-08T06:43:11.304Z{"msg":"Crucible stats registered with UUID: e7a3c4c7-153d-4d49-a9b8-a44325f536f6","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.522786726Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
99942024-02-08T06:43:11.304Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.522815224Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506}
99952024-02-08T06:43:11.305Z{"msg":"e7a3c4c7-153d-4d49-a9b8-a44325f536f6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.522846142Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
99962024-02-08T06:43:11.305Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.52287336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
99972024-02-08T06:43:11.305Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.522919636Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
99982024-02-08T06:43:11.305Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.522989181Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
99992024-02-08T06:43:11.305Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.523052346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100002024-02-08T06:43:11.305Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.523089753Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100012024-02-08T06:43:11.305Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.52313617Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100022024-02-08T06:43:11.305Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.523166447Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"io task","client":"2","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100032024-02-08T06:43:11.305Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.523213464Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"io task","client":"0","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100042024-02-08T06:43:11.305Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.52327158Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"io task","client":"1","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100052024-02-08T06:43:11.305Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.523319056Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"io task","client":"2","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100062024-02-08T06:43:11.305Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.523353873Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"io task","client":"1","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100072024-02-08T06:43:11.305Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.5233968Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"io task","client":"0","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100082024-02-08T06:43:11.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.523835887Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100092024-02-08T06:43:11.305Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.523877214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100102024-02-08T06:43:11.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.523893003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100112024-02-08T06:43:11.306Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.523988746Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100122024-02-08T06:43:11.306Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.524549504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100132024-02-08T06:43:11.306Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.524588371Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100142024-02-08T06:43:11.306Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.524953453Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100152024-02-08T06:43:11.306Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.525061935Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100162024-02-08T06:43:11.306Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.525175387Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100172024-02-08T06:43:11.306Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.525245342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100182024-02-08T06:43:11.306Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.525279289Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100192024-02-08T06:43:11.306Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.525293598Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100202024-02-08T06:43:11.306Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.525308247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100212024-02-08T06:43:11.306Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.525322656Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100222024-02-08T06:43:11.306Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.525650311Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100232024-02-08T06:43:11.306Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526128055Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100242024-02-08T06:43:11.306Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526155053Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100252024-02-08T06:43:11.306Z{"msg":"[0]R flush_numbers[0..12]: [145, 145, 135, 145, 144, 133, 144, 139, 139, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526171302Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100262024-02-08T06:43:11.307Z{"msg":"[0]R generation[0..12]: [91, 91, 88, 91, 90, 86, 90, 88, 88, 89, 89, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.52619218Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100272024-02-08T06:43:11.307Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526206649Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100282024-02-08T06:43:11.307Z{"msg":"[1]R flush_numbers[0..12]: [145, 145, 135, 145, 144, 133, 144, 139, 139, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526227328Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100292024-02-08T06:43:11.307Z{"msg":"[1]R generation[0..12]: [91, 91, 88, 91, 90, 86, 90, 88, 88, 89, 89, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526241117Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100302024-02-08T06:43:11.307Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526261805Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100312024-02-08T06:43:11.307Z{"msg":"[2]R flush_numbers[0..12]: [145, 145, 135, 145, 144, 133, 144, 139, 139, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526276284Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100322024-02-08T06:43:11.307Z{"msg":"[2]R generation[0..12]: [91, 91, 88, 91, 90, 86, 90, 88, 88, 89, 89, 88]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
100332024-02-08T06:43:11.307Z,"time":"2024-02-08T06:43:10.526310182Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100342024-02-08T06:43:11.307Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526348389Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100352024-02-08T06:43:11.307Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526409504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100362024-02-08T06:43:11.307Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526431332Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100372024-02-08T06:43:11.307Z{"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.52646236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100382024-02-08T06:43:11.307Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526484678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100392024-02-08T06:43:11.307Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526529565Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100402024-02-08T06:43:11.307Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526560923Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100412024-02-08T06:43:11.308Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526577022Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100422024-02-08T06:43:11.308Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526606349Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100432024-02-08T06:43:11.308Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526643606Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100442024-02-08T06:43:11.308Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526666915Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100452024-02-08T06:43:11.308Z{"msg":"e7a3c4c7-153d-4d49-a9b8-a44325f536f6 is now active with session: 08ada59e-91eb-4b8e-92f9-7117df99d362","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526686143Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100462024-02-08T06:43:11.308Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526718711Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506,"session_id":"08ada59e-91eb-4b8e-92f9-7117df99d362"}
100472024-02-08T06:43:11.308ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
100482024-02-08T06:43:11.308Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:10.526752948Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506}
100492024-02-08T06:43:11.308ZLoading write count information from file "/tmp/test_repair_verify.data"
100502024-02-08T06:43:11.308ZRun Repair workload
100512024-02-08T06:43:11.308Z01/30 Write block 150 len 4096 data: 5
100522024-02-08T06:43:11.308Z02/30 Read block 87 len 28672
100532024-02-08T06:43:11.308Z03/30 Write block 204 len 16384 data: 7 8 7 8
100542024-02-08T06:43:11.308Z04/30 Read block 486 len 4096
100552024-02-08T06:43:11.308Z05/30 Read block 479 len 24576
100562024-02-08T06:43:11.308Z06/30 Flush
100572024-02-08T06:43:11.309Z07/30 Write block 173 len 12288 data: 9 6 7
100582024-02-08T06:43:11.309Z08/30 Write block 540 len 36864 data: 7 6 6 6 7 6 7 11 11
100592024-02-08T06:43:11.313Z09/30 Flush
100602024-02-08T06:43:11.313Z10/30 Write block 139 len 40960 data: 8 6 6 5 4 4 4 3 3 4
100612024-02-08T06:43:11.313Z11/30 Read block 184 len 12288
100622024-02-08T06:43:11.313Z12/30 Write block 288 len 4096 data: 6
100632024-02-08T06:43:11.313Z13/30 Read block 111 len 8192
100642024-02-08T06:43:11.313Z14/30 Read block 141 len 40960
100652024-02-08T06:43:11.313Z15/30 Write block 74 len 12288 data: 7 5 6
100662024-02-08T06:43:11.313Z16/30 Read block 134 len 4096
100672024-02-08T06:43:11.313Z17/30 Read block 283 len 28672
100682024-02-08T06:43:11.314Z18/30 Write block 51 len 8192 data: 6 6
100692024-02-08T06:43:11.314Z19/30 Write block 482 len 28672 data: 6 5 7 8 10 10 10
100702024-02-08T06:43:11.314Z20/30 Read block 499 len 4096
100712024-02-08T06:43:11.314Z21/30 Write block 499 len 8192 data: 8 8
100722024-02-08T06:43:11.317Z22/30 Write block 469 len 12288 data: 9 8 7
100732024-02-08T06:43:11.318Z23/30 Write block 324 len 4096 data: 9
100742024-02-08T06:43:11.318Z24/30 Read block 114 len 16384
100752024-02-08T06:43:11.318Z25/30 Read block 183 len 12288
100762024-02-08T06:43:11.318Z26/30 Write block 266 len 12288 data: 7 9 10
100772024-02-08T06:43:11.318Z27/30 Read block 584 len 28672
100782024-02-08T06:43:11.318Z28/30 Write block 228 len 4096 data: 14
100792024-02-08T06:43:11.318Z29/30 Read block 391 len 8192
100802024-02-08T06:43:11.323Z30/30 Read block 562 len 8192
100812024-02-08T06:43:11.323Z----------------------------------------------------------------
100822024-02-08T06:43:11.323Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
100832024-02-08T06:43:11.323ZStates: Active Active Active
100842024-02-08T06:43:11.323ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
100852024-02-08T06:43:11.323Z 1 Acked 1000 Write 1 Done Done Done false
100862024-02-08T06:43:11.323Z 2 Acked 1001 Read 7 Done Done Done false
100872024-02-08T06:43:11.323Z 3 Acked 1002 Write 4 Done Done Done false
100882024-02-08T06:43:11.323Z 4 Acked 1003 Read 1 Done Done Done false
100892024-02-08T06:43:11.324Z 5 Acked 1004 Read 6 Done Sent Done false
100902024-02-08T06:43:11.324Z 6 Acked 1005 Flush 0 Done Sent Done false
100912024-02-08T06:43:11.324Z 7 Acked 1006 Write 3 Done Sent Done false
100922024-02-08T06:43:11.324Z 8 Acked 1007 Write 9 Done Sent Done false
100932024-02-08T06:43:11.324Z 9 Acked 1008 Flush 0 Done Sent Done false
100942024-02-08T06:43:11.324Z 10 Acked 1009 Write 10 Done Sent Done false
100952024-02-08T06:43:11.324Z 11 Acked 1010 Read 3 Done Sent Done false
100962024-02-08T06:43:11.324Z 12 Acked 1011 Write 1 Done Sent Done false
100972024-02-08T06:43:11.325Z 13 Acked 1012 Read 2 Done Sent Done false
100982024-02-08T06:43:11.325Z 14 Acked 1013 Read 10 Done Sent Done false
100992024-02-08T06:43:11.325Z 15 Acked 1014 Write 3 Done Sent Done false
101002024-02-08T06:43:11.325Z 16 Acked 1015 Read 1 Done Sent Done false
101012024-02-08T06:43:11.325Z 17 Acked 1016 Read 7 Done Sent Done false
101022024-02-08T06:43:11.325Z 18 Acked 1017 Write 2 Done Sent Done false
101032024-02-08T06:43:11.325Z 19 Acked 1018 Write 7 Done Sent Done false
101042024-02-08T06:43:11.325Z 20 Acked 1019 Read 1 Done Sent Done false
101052024-02-08T06:43:11.325Z 21 Acked 1020 Write 2 Done Sent Done false
101062024-02-08T06:43:11.326Z 22 Acked 1021 Write 3 Done Sent Done false
101072024-02-08T06:43:11.326Z 23 Acked 1022 Write 1 Done Sent Done false
101082024-02-08T06:43:11.326Z 24 Acked 1023 Read 4 Done Sent Done false
101092024-02-08T06:43:11.326Z 25 Acked 1024 Read 3 Done Sent Done false
101102024-02-08T06:43:11.326Z 26 Acked 1025 Write 3 Done Sent Done false
101112024-02-08T06:43:11.326Z 27 Acked 1026 Read 7 Done Sent Done false
101122024-02-08T06:43:11.326Z 28 Acked 1027 Write 1 Done Sent Done false
101132024-02-08T06:43:11.326Z 29 Acked 1028 Read 2 Done Sent Done false
101142024-02-08T06:43:11.327Z 30 Acked 1029 Read 2 Done Sent Done false
101152024-02-08T06:43:11.327Z STATES DS:0 DS:1 DS:2 TOTAL
101162024-02-08T06:43:11.327Z New 0 0 0 0
101172024-02-08T06:43:11.327Z Sent 0 26 0 26
101182024-02-08T06:43:11.327Z Done 30 4 30 64
101192024-02-08T06:43:11.327Z Skipped 0 0 0 0
101202024-02-08T06:43:11.327Z Error 0 0 0 0
101212024-02-08T06:43:11.327ZLast Flush: 1008 0 1008
101222024-02-08T06:43:11.327ZDownstairs last five completed:
101232024-02-08T06:43:11.328ZUpstairs last five completed: 30 29 28 27 26
101242024-02-08T06:43:11.328Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:10.553123248Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1506}
101252024-02-08T06:43:11.328ZWrote out file "/tmp/test_repair_verify.data"
101262024-02-08T06:43:11.333Z
101272024-02-08T06:43:11.333ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
101282024-02-08T06:43:11.345ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101292024-02-08T06:43:11.345Z 0 000-019 91 91 91 145 145 145 F F F
101302024-02-08T06:43:11.345Z 1 020-039 91 91 91 145 145 145 F F F
101312024-02-08T06:43:11.345Z 2 040-059 88 88 88 135 135 135 T F T <---
101322024-02-08T06:43:11.345Z 3 060-079 91 91 91 145 145 145 T F T <---
101332024-02-08T06:43:11.345Z 4 080-099 90 90 90 144 144 144 F F F
101342024-02-08T06:43:11.345Z 5 100-119 86 86 86 133 133 133 F F F
101352024-02-08T06:43:11.345Z 6 120-139 90 90 90 144 144 144 T F T <---
101362024-02-08T06:43:11.350Z 7 140-159 92 88 92 146 139 146 T T T <---
101372024-02-08T06:43:11.350Z 8 160-179 92 88 92 147 139 147 F F F <---
101382024-02-08T06:43:11.350Z 9 180-199 89 89 89 140 140 140 F F F
101392024-02-08T06:43:11.350Z 10 200-219 92 89 92 146 140 146 F T F <---
101402024-02-08T06:43:11.350Z 11 220-239 88 88 88 138 138 138 T F T <---
101412024-02-08T06:43:11.350Z 12 240-259 85 85 85 130 130 130 F F F
101422024-02-08T06:43:11.351Z 13 260-279 85 85 85 130 130 130 T F T <---
101432024-02-08T06:43:11.351Z 14 280-299 86 86 86 132 132 132 T F T <---
101442024-02-08T06:43:11.351Z 15 300-319 90 90 90 144 144 144 F F F
101452024-02-08T06:43:11.351Z 16 320-339 91 91 91 145 145 145 T F T <---
101462024-02-08T06:43:11.351Z 17 340-359 80 80 80 123 123 123 F F F
101472024-02-08T06:43:11.351Z 18 360-379 90 90 90 143 143 143 F F F
101482024-02-08T06:43:11.351Z 19 380-399 86 86 86 133 133 133 F F F
101492024-02-08T06:43:11.351Z 20 400-419 91 91 91 145 145 145 F F F
101502024-02-08T06:43:11.351Z 21 420-439 90 90 90 141 141 141 F F F
101512024-02-08T06:43:11.351Z 22 440-459 90 90 90 144 144 144 F F F
101522024-02-08T06:43:11.351Z 23 460-479 86 86 86 133 133 133 T F T <---
101532024-02-08T06:43:11.351Z 24 480-499 89 89 89 140 140 140 T F T <---
101542024-02-08T06:43:11.351Z 25 500-519 81 81 81 124 124 124 T F T <---
101552024-02-08T06:43:11.351Z 26 520-539 81 81 81 124 124 124 F F F
101562024-02-08T06:43:11.351Z 27 540-559 92 91 92 147 145 147 F F F <---
101572024-02-08T06:43:11.352Z 28 560-579 91 91 91 145 145 145 F F F
101582024-02-08T06:43:11.352Z 29 580-599 74 74 74 118 118 118 F F F
101592024-02-08T06:43:11.352ZMax gen: 92, Max flush: 147
101602024-02-08T06:43:11.352ZError: Difference in extent metadata found!
101612024-02-08T06:43:11.352ZA Difference in extent metadata is expected here
101622024-02-08T06:43:11.352ZOn loop 46
101632024-02-08T06:43:11.352Z
101642024-02-08T06:43:11.355ZVerifying data now
101652024-02-08T06:43:17.051ZNo change after verify
101662024-02-08T06:43:17.051ZLoop: 46 Downstairs dump after verify (and repair):
101672024-02-08T06:43:17.067ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101682024-02-08T06:43:17.067Z 0 000-019 91 91 91 145 145 145 F F F
101692024-02-08T06:43:17.067Z 1 020-039 91 91 91 145 145 145 F F F
101702024-02-08T06:43:17.067Z 2 040-059 93 93 93 148 148 148 F F F
101712024-02-08T06:43:17.067Z 3 060-079 93 93 93 148 148 148 F F F
101722024-02-08T06:43:17.067Z 4 080-099 90 90 90 144 144 144 F F F
101732024-02-08T06:43:17.067Z 5 100-119 86 86 86 133 133 133 F F F
101742024-02-08T06:43:17.067Z 6 120-139 93 93 93 148 148 148 F F F
101752024-02-08T06:43:17.068Z 7 140-159 93 93 93 148 148 148 F F F
101762024-02-08T06:43:17.068Z 8 160-179 92 92 92 147 147 147 F F F
101772024-02-08T06:43:17.068Z 9 180-199 89 89 89 140 140 140 F F F
101782024-02-08T06:43:17.068Z 10 200-219 92 92 92 146 146 146 F F F
101792024-02-08T06:43:17.071Z 11 220-239 93 93 93 148 148 148 F F F
101802024-02-08T06:43:17.071Z 12 240-259 85 85 85 130 130 130 F F F
101812024-02-08T06:43:17.071Z 13 260-279 93 93 93 148 148 148 F F F
101822024-02-08T06:43:17.071Z 14 280-299 93 93 93 148 148 148 F F F
101832024-02-08T06:43:17.071Z 15 300-319 90 90 90 144 144 144 F F F
101842024-02-08T06:43:17.071Z 16 320-339 93 93 93 148 148 148 F F F
101852024-02-08T06:43:17.071Z 17 340-359 80 80 80 123 123 123 F F F
101862024-02-08T06:43:17.072Z 18 360-379 90 90 90 143 143 143 F F F
101872024-02-08T06:43:17.072Z 19 380-399 86 86 86 133 133 133 F F F
101882024-02-08T06:43:17.072Z 20 400-419 91 91 91 145 145 145 F F F
101892024-02-08T06:43:17.072Z 21 420-439 90 90 90 141 141 141 F F F
101902024-02-08T06:43:17.072Z 22 440-459 90 90 90 144 144 144 F F F
101912024-02-08T06:43:17.072Z 23 460-479 93 93 93 148 148 148 F F F
101922024-02-08T06:43:17.072Z 24 480-499 93 93 93 148 148 148 F F F
101932024-02-08T06:43:17.072Z 25 500-519 93 93 93 148 148 148 F F F
101942024-02-08T06:43:17.072Z 26 520-539 81 81 81 124 124 124 F F F
101952024-02-08T06:43:17.072Z 27 540-559 92 92 92 147 147 147 F F F
101962024-02-08T06:43:17.072Z 28 560-579 91 91 91 145 145 145 F F F
101972024-02-08T06:43:17.072Z 29 580-599 74 74 74 118 118 118 F F F
101982024-02-08T06:43:17.072ZMax gen: 93, Max flush: 148
101992024-02-08T06:43:17.072Z
102002024-02-08T06:43:17.072ZBegin loop 47
102012024-02-08T06:43:17.073ZDownstairs to restart: 1
102022024-02-08T06:43:17.087Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.320680682Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514}
102032024-02-08T06:43:17.087Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.320916294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514}
102042024-02-08T06:43:17.092Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.320933563Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514}
102052024-02-08T06:43:17.093Z{"msg":"Crucible 52ab4fa3-5e5b-46f4-a5a3-8b205f0038d3 has session id: 1d474022-857c-4696-8f0a-658d3ebbc1ad","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.32096516Z","Crucible runtime is spawned
102062024-02-08T06:43:17.093Zhostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102072024-02-08T06:43:17.093Z{"msg":"Upstairs opts: Upstairs UUID: 52ab4fa3-5e5b-46f4-a5a3-8b205f0038d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321011637Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102082024-02-08T06:43:17.093Z{"msg":"Crucible stats registered with UUID: 52ab4fa3-5e5b-46f4-a5a3-8b205f0038d3","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321048254Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102092024-02-08T06:43:17.093Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321085071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514}
102102024-02-08T06:43:17.093Z{"msg":"52ab4fa3-5e5b-46f4-a5a3-8b205f0038d3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321161836Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102112024-02-08T06:43:17.093Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321187764Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102122024-02-08T06:43:17.093Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321211442Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102132024-02-08T06:43:17.093Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321246189Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102142024-02-08T06:43:17.093Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321270968Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102152024-02-08T06:43:17.093Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321304925Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102162024-02-08T06:43:17.093Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321353031Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102172024-02-08T06:43:17.093Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321389719Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102182024-02-08T06:43:17.093Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321423666Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102192024-02-08T06:43:17.093Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321551956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102202024-02-08T06:43:17.094Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321587054Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102212024-02-08T06:43:17.094Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321614482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102222024-02-08T06:43:17.094Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.321648939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102232024-02-08T06:43:17.094Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.322235365Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102242024-02-08T06:43:17.094Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.322264833Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102252024-02-08T06:43:17.094Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.322281072Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102262024-02-08T06:43:17.094Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.322316099Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102272024-02-08T06:43:17.094Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.322865118Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102282024-02-08T06:43:17.094Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.322929083Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102292024-02-08T06:43:17.094Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.323487481Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102302024-02-08T06:43:17.094Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.323538117Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102312024-02-08T06:43:17.094Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.323578974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102322024-02-08T06:43:17.094Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.323613092Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102332024-02-08T06:43:17.094Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.323647139Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102342024-02-08T06:43:17.094Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.323691426Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102352024-02-08T06:43:17.094Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.323806347Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102362024-02-08T06:43:17.095Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.323848734Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102372024-02-08T06:43:17.095Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.32430786Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102382024-02-08T06:43:17.100Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324777774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102392024-02-08T06:43:17.100Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324808342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102402024-02-08T06:43:17.100Z{"msg":"[0]R flush_numbers[0..12]: [145, 145, 148, 148, 144, 133, 148, 148, 147, 140, 146, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324823411Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102412024-02-08T06:43:17.100Z{"msg":"[0]R generation[0..12]: [91, 91, 93, 93, 90, 86, 93, 93, 92, 89, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.32483762Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102422024-02-08T06:43:17.100Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324859538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102432024-02-08T06:43:17.100Z{"msg":"[1]R flush_numbers[0..12]: [145, 145, 148, 148, 144, 133, 148, 148, 147, 140, 146, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324873967Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102442024-02-08T06:43:17.100Z{"msg":"[1]R generation[0..12]: [91, 91, 93, 93, 90, 86, 93, 93, 92, 89, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324894205Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102452024-02-08T06:43:17.100Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324908974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102462024-02-08T06:43:17.100Z{"msg":"[2]R flush_numbers[0..12]: [145, 145, 148, 148, 144, 133, 148, 148, 147, 140, 146, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324924393Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102472024-02-08T06:43:17.100Z{"msg":"[2]R generation[0..12]: [91, 91, 93, 93, 90, 86, 93, 93, 92, 89, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324944952Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","Wait for a query_work_queue command to finish before sending IO
102482024-02-08T06:43:17.100Z":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102492024-02-08T06:43:17.100Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.32496302Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102502024-02-08T06:43:17.100Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324983129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102512024-02-08T06:43:17.101Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.324996728Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102522024-02-08T06:43:17.101Z{"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.325009707Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102532024-02-08T06:43:17.101Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.325035635Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102542024-02-08T06:43:17.101Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.325053304Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102552024-02-08T06:43:17.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.325069922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102562024-02-08T06:43:17.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.325090601Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102572024-02-08T06:43:17.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.3251043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102582024-02-08T06:43:17.101Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.325124658Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102592024-02-08T06:43:17.101Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.325137477Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102602024-02-08T06:43:17.101Z{"msg":"52ab4fa3-5e5b-46f4-a5a3-8b205f0038d3 is now active with session: 1d474022-857c-4696-8f0a-658d3ebbc1ad","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.325156906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102612024-02-08T06:43:17.101Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.325169255Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514,"session_id":"1d474022-857c-4696-8f0a-658d3ebbc1ad"}
102622024-02-08T06:43:17.101Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:16.325187254Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514}
102632024-02-08T06:43:17.101ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
102642024-02-08T06:43:17.101ZLoading write count information from file "/tmp/test_repair_verify.data"
102652024-02-08T06:43:17.101ZRun Repair workload
102662024-02-08T06:43:17.102Z01/30 Write block 406 len 4096 data: 9
102672024-02-08T06:43:17.102Z02/30 Write block 95 len 12288 data: 14 14 14
102682024-02-08T06:43:17.102Z03/30 Read block 234 len 20480
102692024-02-08T06:43:17.102Z04/30 Read block 167 len 8192
102702024-02-08T06:43:17.102Z05/30 Write block 378 len 12288 data: 6 6 5
102712024-02-08T06:43:17.102Z06/30 Write block 596 len 8192 data: 7 6
102722024-02-08T06:43:17.102Z07/30 Read block 92 len 32768
102732024-02-08T06:43:17.102Z08/30 Read block 247 len 8192
102742024-02-08T06:43:17.102Z09/30 Read block 397 len 16384
102752024-02-08T06:43:17.106Z10/30 Read block 386 len 24576
102762024-02-08T06:43:17.106Z11/30 Write block 215 len 12288 data: 7 7 8
102772024-02-08T06:43:17.106Z12/30 Read block 437 len 8192
102782024-02-08T06:43:17.106Z13/30 Read block 320 len 20480
102792024-02-08T06:43:17.107Z14/30 Flush
102802024-02-08T06:43:17.107Z15/30 Write block 279 len 32768 data: 5 4 4 4 5 3 5 4
102812024-02-08T06:43:17.107Z16/30 Write block 367 len 4096 data: 7
102822024-02-08T06:43:17.107Z17/30 Read block 229 len 16384
102832024-02-08T06:43:17.110Z18/30 Flush
102842024-02-08T06:43:17.110Z19/30 Write block 531 len 8192 data: 10 10
102852024-02-08T06:43:17.110Z20/30 Write block 537 len 12288 data: 9 7 5
102862024-02-08T06:43:17.110Z21/30 Flush
102872024-02-08T06:43:17.113Z22/30 Write block 222 len 28672 data: 12 12 14 13 13 13 15
102882024-02-08T06:43:17.113Z23/30 Flush
102892024-02-08T06:43:17.116Z24/30 Write block 288 len 40960 data: 7 7 9 9 7 6 4 4 4 3
102902024-02-08T06:43:17.116Z25/30 Read block 503 len 28672
102912024-02-08T06:43:17.119Z26/30 Read block 345 len 32768
102922024-02-08T06:43:17.119Z27/30 Read block 502 len 16384
102932024-02-08T06:43:17.125Z28/30 Write block 10 len 40960 data: 8 6 6 7 6 5 5 6 7 7
102942024-02-08T06:43:17.125Z29/30 Read block 334 len 16384
102952024-02-08T06:43:17.125Z30/30 Write block 301 len 24576 data: 11 11 10 9 9 6
102962024-02-08T06:43:17.125Z----------------------------------------------------------------
102972024-02-08T06:43:17.125Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
102982024-02-08T06:43:17.125ZStates: Active Active Active
102992024-02-08T06:43:17.125ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
103002024-02-08T06:43:17.125Z 1 Acked 1000 Write 1 Done Sent Done false
103012024-02-08T06:43:17.125Z 2 Acked 1001 Write 3 Done Sent Done false
103022024-02-08T06:43:17.125Z 3 Acked 1002 Read 5 Done Sent Done false
103032024-02-08T06:43:17.125Z 4 Acked 1003 Read 2 Done Sent Done false
103042024-02-08T06:43:17.126Z 5 Acked 1004 Write 3 Done Sent Done false
103052024-02-08T06:43:17.126Z 6 Acked 1005 Write 2 Done Sent Done false
103062024-02-08T06:43:17.126Z 7 Acked 1006 Read 8 Done Sent Done false
103072024-02-08T06:43:17.126Z 8 Acked 1007 Read 2 Done Sent Done false
103082024-02-08T06:43:17.126Z 9 Acked 1008 Read 4 Done Sent Done false
103092024-02-08T06:43:17.126Z 10 Acked 1009 Read 6 Done Sent Done false
103102024-02-08T06:43:17.126Z 11 Acked 1010 Write 3 Done Sent Done false
103112024-02-08T06:43:17.126Z 12 Acked 1011 Read 2 Done Sent Done false
103122024-02-08T06:43:17.126Z 13 Acked 1012 Read 5 Done Sent Done false
103132024-02-08T06:43:17.126Z 14 Acked 1013 Flush 0 Done Sent Done false
103142024-02-08T06:43:17.126Z 15 Acked 1014 Write 8 Done Sent Done false
103152024-02-08T06:43:17.126Z 16 Acked 1015 Write 1 Done Sent Done false
103162024-02-08T06:43:17.126Z 17 Acked 1016 Read 4 Done Sent Done false
103172024-02-08T06:43:17.126Z 18 Acked 1017 Flush 0 Done Sent Done false
103182024-02-08T06:43:17.126Z 19 Acked 1018 Write 2 Done Sent Done false
103192024-02-08T06:43:17.126Z 20 Acked 1019 Write 3 Done Sent Done false
103202024-02-08T06:43:17.127Z 21 Acked 1020 Flush 0 Done Sent Done false
103212024-02-08T06:43:17.127Z 22 Acked 1021 Write 7 Done Sent Done false
103222024-02-08T06:43:17.127Z 23 Acked 1022 Flush 0 Done Sent Done false
103232024-02-08T06:43:17.127Z 24 Acked 1023 Write 10 Done Sent Done false
103242024-02-08T06:43:17.127Z 25 Acked 1024 Read 7 Done Sent Done false
103252024-02-08T06:43:17.127Z 26 Acked 1025 Read 8 Done Sent Done false
103262024-02-08T06:43:17.127Z 27 Acked 1026 Read 4 Done Sent Done false
103272024-02-08T06:43:17.127Z 28 Acked 1027 Write 10 Done Sent Done false
103282024-02-08T06:43:17.127Z 29 Acked 1028 Read 4 Done Sent Done false
103292024-02-08T06:43:17.127Z 30 Acked 1029 Write 6 Sent Sent Sent false
103302024-02-08T06:43:17.127Z STATES DS:0 DS:1 DS:2 TOTAL
103312024-02-08T06:43:17.127Z New 0 0 0 0
103322024-02-08T06:43:17.127Z Sent 1 30 1 32
103332024-02-08T06:43:17.127Z Done 29 0 29 58
103342024-02-08T06:43:17.127Z Skipped 0 0 0 0
103352024-02-08T06:43:17.127Z Error 0 0 0 0
103362024-02-08T06:43:17.128ZLast Flush: 1022 0 1022
103372024-02-08T06:43:17.128ZDownstairs last five completed:
103382024-02-08T06:43:17.128ZUpstairs last five completed: 30 29 28 27 26
103392024-02-08T06:43:17.131Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:16.35560435Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1514}
103402024-02-08T06:43:17.131ZWrote out file "/tmp/test_repair_verify.data"
103412024-02-08T06:43:17.131Z
103422024-02-08T06:43:17.131ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
103432024-02-08T06:43:17.145ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103442024-02-08T06:43:17.145Z 0 000-019 91 91 91 145 145 145 T F T <---
103452024-02-08T06:43:17.145Z 1 020-039 91 91 91 145 145 145 F F F
103462024-02-08T06:43:17.145Z 2 040-059 93 93 93 148 148 148 F F F
103472024-02-08T06:43:17.145Z 3 060-079 93 93 93 148 148 148 F F F
103482024-02-08T06:43:17.145Z 4 080-099 94 90 94 149 144 149 F F F <---
103492024-02-08T06:43:17.145Z 5 100-119 86 86 86 133 133 133 F F F
103502024-02-08T06:43:17.145Z 6 120-139 93 93 93 148 148 148 F F F
103512024-02-08T06:43:17.145Z 7 140-159 93 93 93 148 148 148 F F F
103522024-02-08T06:43:17.145Z 8 160-179 92 92 92 147 147 147 F F F
103532024-02-08T06:43:17.145Z 9 180-199 89 89 89 140 140 140 F F F
103542024-02-08T06:43:17.145Z 10 200-219 94 92 94 149 146 149 F F F <---
103552024-02-08T06:43:17.149Z 11 220-239 94 93 94 152 148 152 F F F <---
103562024-02-08T06:43:17.149Z 12 240-259 85 85 85 130 130 130 F F F
103572024-02-08T06:43:17.149Z 13 260-279 94 93 94 150 148 150 F F F <---
103582024-02-08T06:43:17.149Z 14 280-299 94 93 94 150 148 150 T F T <---
103592024-02-08T06:43:17.149Z 15 300-319 90 90 90 144 144 144 T F T <---
103602024-02-08T06:43:17.149Z 16 320-339 93 93 93 148 148 148 F F F
103612024-02-08T06:43:17.149Z 17 340-359 80 80 80 123 123 123 F F F
103622024-02-08T06:43:17.149Z 18 360-379 94 90 94 150 143 150 F F F <---
103632024-02-08T06:43:17.149Z 19 380-399 94 86 94 149 133 149 F F F <---
103642024-02-08T06:43:17.149Z 20 400-419 94 91 94 149 145 149 F F F <---
103652024-02-08T06:43:17.149Z 21 420-439 90 90 90 141 141 141 F F F
103662024-02-08T06:43:17.149Z 22 440-459 90 90 90 144 144 144 F F F
103672024-02-08T06:43:17.150Z 23 460-479 93 93 93 148 148 148 F F F
103682024-02-08T06:43:17.150Z 24 480-499 93 93 93 148 148 148 F F F
103692024-02-08T06:43:17.150Z 25 500-519 93 93 93 148 148 148 F F F
103702024-02-08T06:43:17.150Z 26 520-539 94 81 94 151 124 151 F F F <---
103712024-02-08T06:43:17.150Z 27 540-559 92 92 92 147 147 147 F F F
103722024-02-08T06:43:17.150Z 28 560-579 91 91 91 145 145 145 F F F
103732024-02-08T06:43:17.150Z 29 580-599 94 74 94 149 118 149 F F F <---
103742024-02-08T06:43:17.150ZMax gen: 94, Max flush: 152
103752024-02-08T06:43:17.150ZError: Difference in extent metadata found!
103762024-02-08T06:43:17.150ZA Difference in extent metadata is expected here
103772024-02-08T06:43:17.150ZOn loop 47
103782024-02-08T06:43:17.150Z
103792024-02-08T06:43:17.155ZVerifying data now
103802024-02-08T06:43:21.751ZNo change after verify
103812024-02-08T06:43:21.752ZLoop: 47 Downstairs dump after verify (and repair):
103822024-02-08T06:43:21.767ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103832024-02-08T06:43:21.767Z 0 000-019 95 95 95 153 153 153 F F F
103842024-02-08T06:43:21.767Z 1 020-039 91 91 91 145 145 145 F F F
103852024-02-08T06:43:21.767Z 2 040-059 93 93 93 148 148 148 F F F
103862024-02-08T06:43:21.767Z 3 060-079 93 93 93 148 148 148 F F F
103872024-02-08T06:43:21.771Z 4 080-099 94 94 94 149 149 149 F F F
103882024-02-08T06:43:21.771Z 5 100-119 86 86 86 133 133 133 F F F
103892024-02-08T06:43:21.771Z 6 120-139 93 93 93 148 148 148 F F F
103902024-02-08T06:43:21.771Z 7 140-159 93 93 93 148 148 148 F F F
103912024-02-08T06:43:21.771Z 8 160-179 92 92 92 147 147 147 F F F
103922024-02-08T06:43:21.771Z 9 180-199 89 89 89 140 140 140 F F F
103932024-02-08T06:43:21.771Z 10 200-219 94 94 94 149 149 149 F F F
103942024-02-08T06:43:21.772Z 11 220-239 94 94 94 152 152 152 F F F
103952024-02-08T06:43:21.772Z 12 240-259 85 85 85 130 130 130 F F F
103962024-02-08T06:43:21.772Z 13 260-279 94 94 94 150 150 150 F F F
103972024-02-08T06:43:21.772Z 14 280-299 95 95 95 153 153 153 F F F
103982024-02-08T06:43:21.772Z 15 300-319 95 95 95 153 153 153 F F F
103992024-02-08T06:43:21.772Z 16 320-339 93 93 93 148 148 148 F F F
104002024-02-08T06:43:21.772Z 17 340-359 80 80 80 123 123 123 F F F
104012024-02-08T06:43:21.772Z 18 360-379 94 94 94 150 150 150 F F F
104022024-02-08T06:43:21.772Z 19 380-399 94 94 94 149 149 149 F F F
104032024-02-08T06:43:21.772Z 20 400-419 94 94 94 149 149 149 F F F
104042024-02-08T06:43:21.772Z 21 420-439 90 90 90 141 141 141 F F F
104052024-02-08T06:43:21.772Z 22 440-459 90 90 90 144 144 144 F F F
104062024-02-08T06:43:21.772Z 23 460-479 93 93 93 148 148 148 F F F
104072024-02-08T06:43:21.772Z 24 480-499 93 93 93 148 148 148 F F F
104082024-02-08T06:43:21.772Z 25 500-519 93 93 93 148 148 148 F F F
104092024-02-08T06:43:21.773Z 26 520-539 94 94 94 151 151 151 F F F
104102024-02-08T06:43:21.773Z 27 540-559 92 92 92 147 147 147 F F F
104112024-02-08T06:43:21.773Z 28 560-579 91 91 91 145 145 145 F F F
104122024-02-08T06:43:21.773Z 29 580-599 94 94 94 149 149 149 F F F
104132024-02-08T06:43:21.773ZMax gen: 95, Max flush: 153
104142024-02-08T06:43:21.773Z
104152024-02-08T06:43:21.773ZBegin loop 48
104162024-02-08T06:43:21.773ZDownstairs to restart: 2
104172024-02-08T06:43:21.789Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.021632943Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522}
104182024-02-08T06:43:21.789Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}"Crucible runtime is spawned
104192024-02-08T06:43:21.789Z,"v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.021913972Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522}
104202024-02-08T06:43:21.794Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.02194201Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522}
104212024-02-08T06:43:21.795Z{"msg":"Crucible da3eece9-a81a-4040-8450-ec78b29536bb has session id: c2a09fcf-19ea-494c-b5f3-0bf5658fd81b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.021976038Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104222024-02-08T06:43:21.795Z{"msg":"Upstairs opts: Upstairs UUID: da3eece9-a81a-4040-8450-ec78b29536bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022019924Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104232024-02-08T06:43:21.795Z{"msg":"Crucible stats registered with UUID: da3eece9-a81a-4040-8450-ec78b29536bb","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022047952Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104242024-02-08T06:43:21.795Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.0220782Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522}
104252024-02-08T06:43:21.795Z{"msg":"da3eece9-a81a-4040-8450-ec78b29536bb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022092109Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104262024-02-08T06:43:21.795Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022109858Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104272024-02-08T06:43:21.795Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022152254Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104282024-02-08T06:43:21.795Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022182582Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104292024-02-08T06:43:21.795Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022227609Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104302024-02-08T06:43:21.795Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022253727Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104312024-02-08T06:43:21.795Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022308943Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104322024-02-08T06:43:21.795Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022355059Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"io task","client":"0","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104332024-02-08T06:43:21.795Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022394186Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"io task","client":"1","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104342024-02-08T06:43:21.795Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022420644Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104352024-02-08T06:43:21.795Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022456401Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"io task","client":"1","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104362024-02-08T06:43:21.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.02248415Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"io task","client":"0","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104372024-02-08T06:43:21.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.022511507Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104382024-02-08T06:43:21.796Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.023162708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104392024-02-08T06:43:21.796Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.023211995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104402024-02-08T06:43:21.796Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.023257441Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104412024-02-08T06:43:21.796Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.023295118Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104422024-02-08T06:43:21.796Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.0238104Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104432024-02-08T06:43:21.796Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.023860496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104442024-02-08T06:43:21.796Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.02447354Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104452024-02-08T06:43:21.796Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.024512227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104462024-02-08T06:43:21.796Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.024530096Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104472024-02-08T06:43:21.796Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.024551084Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104482024-02-08T06:43:21.796Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.024566183Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104492024-02-08T06:43:21.796Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.024587101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104502024-02-08T06:43:21.796Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.024768438Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104512024-02-08T06:43:21.796Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.024795986Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104522024-02-08T06:43:21.797Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025297678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104532024-02-08T06:43:21.801Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025798481Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104542024-02-08T06:43:21.802Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025835848Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104552024-02-08T06:43:21.802Z{"msg":"[0]R flush_numbers[0..12]: [153, 145, 148, 148, 149, 133, 148, 148, 147, 140, 149, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025853226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104562024-02-08T06:43:21.802Z{"msg":"[0]R generation[0..12]: [95, 91, 93, 93, 94, 86, 93, 93, 92, 89, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025876345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104572024-02-08T06:43:21.802Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025898473Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104582024-02-08T06:43:21.802Z{"msg":"[1]R flush_numbers[0..12]: [153, 145, 148, 148, 149, 133, 148, 148, 147, 140, 149, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025913302Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104592024-02-08T06:43:21.802Z{"msg":"[1]R generation[0..12]: [95, 91, 93, 93, 94, 86, 93, 93, 92, 89, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025927981Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104602024-02-08T06:43:21.802Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025948949Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104612024-02-08T06:43:21.802Z{"msg":"[2]R flush_numbers[0..12]: [153, 145, 148, 148, 149, 133, 148, 148, 147, 140, 149, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025963468Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104622024-02-08T06:43:21.802Z{"msg":"[2]R generation[0..12]: [95, 91, 93, 93, 94, 86, 93, 93, 92, 89, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025983977Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104632024-02-08T06:43:21.802Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.025999866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104642024-02-08T06:43:21.802Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
104652024-02-08T06:43:21.802Z,"time":"2024-02-08T06:43:21.026031153Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104662024-02-08T06:43:21.802Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026047432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104672024-02-08T06:43:21.802Z{"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.02606778Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104682024-02-08T06:43:21.803Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026081509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104692024-02-08T06:43:21.803Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026101238Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104702024-02-08T06:43:21.803Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026113847Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104712024-02-08T06:43:21.803Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026134296Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104722024-02-08T06:43:21.803Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026149534Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104732024-02-08T06:43:21.803Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026169543Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104742024-02-08T06:43:21.803Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026182332Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104752024-02-08T06:43:21.803Z{"msg":"da3eece9-a81a-4040-8450-ec78b29536bb is now active with session: c2a09fcf-19ea-494c-b5f3-0bf5658fd81b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026195501Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104762024-02-08T06:43:21.803Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026214099Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522,"session_id":"c2a09fcf-19ea-494c-b5f3-0bf5658fd81b"}
104772024-02-08T06:43:21.803Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:21.026226678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522}
104782024-02-08T06:43:21.803ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
104792024-02-08T06:43:21.803ZLoading write count information from file "/tmp/test_repair_verify.data"
104802024-02-08T06:43:21.803ZRun Repair workload
104812024-02-08T06:43:21.804Z01/30 Flush
104822024-02-08T06:43:21.804Z02/30 Write block 415 len 32768 data: 6 5 6 8 7 7 6 7
104832024-02-08T06:43:21.804Z03/30 Read block 16 len 16384
104842024-02-08T06:43:21.804Z04/30 Write block 98 len 32768 data: 13 13 14 12 9 10 10 8
104852024-02-08T06:43:21.804Z05/30 Read block 35 len 12288
104862024-02-08T06:43:21.804Z06/30 Write block 261 len 24576 data: 12 10 7 6 7 8
104872024-02-08T06:43:21.804Z07/30 Read block 228 len 20480
104882024-02-08T06:43:21.810Z08/30 Write block 261 len 20480 data: 13 11 8 7 8
104892024-02-08T06:43:21.811Z09/30 Read block 308 len 36864
104902024-02-08T06:43:21.811Z10/30 Flush
104912024-02-08T06:43:21.811Z11/30 Write block 585 len 36864 data: 6 6 7 7 10 10 9 8 9
104922024-02-08T06:43:21.811Z12/30 Write block 116 len 20480 data: 8 6 4 3 3
104932024-02-08T06:43:21.811Z13/30 Write block 306 len 36864 data: 7 4 3 4 5 5 6 7 7
104942024-02-08T06:43:21.814Z14/30 Write block 460 len 36864 data: 6 7 8 8 6 5 6 8 8
104952024-02-08T06:43:21.814Z15/30 Write block 149 len 20480 data: 6 6 7 6 6
104962024-02-08T06:43:21.814Z16/30 Write block 218 len 8192 data: 7 10
104972024-02-08T06:43:21.814Z17/30 Read block 326 len 32768
104982024-02-08T06:43:21.814Z18/30 Write block 409 len 20480 data: 7 6 5 6 6
104992024-02-08T06:43:21.814Z19/30 Write block 276 len 36864 data: 7 8 4 6 5 5 5 6 4
105002024-02-08T06:43:21.814Z20/30 Write block 501 len 24576 data: 7 5 4 7 5 5
105012024-02-08T06:43:21.814Z21/30 Read block 148 len 4096
105022024-02-08T06:43:21.817Z22/30 Write block 173 len 16384 data: 10 7 8 7
105032024-02-08T06:43:21.818Z23/30 Read block 102 len 20480
105042024-02-08T06:43:21.818Z24/30 Write block 32 len 28672 data: 7 8 7 7 6 5 7
105052024-02-08T06:43:21.818Z25/30 Read block 213 len 40960
105062024-02-08T06:43:21.821Z26/30 Write block 316 len 8192 data: 5 5
105072024-02-08T06:43:21.821Z27/30 Write block 104 len 24576 data: 11 9 8 7 8 6
105082024-02-08T06:43:21.821Z28/30 Write block 392 len 12288 data: 7 4 5
105092024-02-08T06:43:21.821Z29/30 Read block 368 len 28672
105102024-02-08T06:43:21.824Z30/30 Write block 206 len 32768 data: 8 9 6 7 7 10 12 11
105112024-02-08T06:43:21.824Z----------------------------------------------------------------
105122024-02-08T06:43:21.825Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:20
105132024-02-08T06:43:21.825ZStates: Active Active Active
105142024-02-08T06:43:21.825ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
105152024-02-08T06:43:21.825Z 11 Acked 1010 Write 9 Done Done Done false
105162024-02-08T06:43:21.825Z 12 Acked 1011 Write 5 Done Done Done false
105172024-02-08T06:43:21.825Z 13 Acked 1012 Write 9 Done Done Sent false
105182024-02-08T06:43:21.825Z 14 Acked 1013 Write 9 Done Done Sent false
105192024-02-08T06:43:21.825Z 15 Acked 1014 Write 5 Done Done Sent false
105202024-02-08T06:43:21.825Z 16 Acked 1015 Write 2 Done Done Sent false
105212024-02-08T06:43:21.825Z 17 Acked 1016 Read 8 Done Done Sent false
105222024-02-08T06:43:21.825Z 18 Acked 1017 Write 5 Done Done Sent false
105232024-02-08T06:43:21.825Z 19 Acked 1018 Write 9 Done Done Sent false
105242024-02-08T06:43:21.825Z 20 Acked 1019 Write 6 Done Done Sent false
105252024-02-08T06:43:21.825Z 21 Acked 1020 Read 1 Done Done Sent false
105262024-02-08T06:43:21.825Z 22 Acked 1021 Write 4 Done Done Sent false
105272024-02-08T06:43:21.826Z 23 Acked 1022 Read 5 Done Done Sent false
105282024-02-08T06:43:21.826Z 24 Acked 1023 Write 7 Done Done Sent false
105292024-02-08T06:43:21.826Z 25 Acked 1024 Read 10 Done Done Sent false
105302024-02-08T06:43:21.826Z 26 Acked 1025 Write 2 Done Done Sent false
105312024-02-08T06:43:21.826Z 27 Acked 1026 Write 6 Done Done Sent false
105322024-02-08T06:43:21.826Z 28 Acked 1027 Write 3 Done Done Sent false
105332024-02-08T06:43:21.826Z 29 Acked 1028 Read 7 Done Done Sent false
105342024-02-08T06:43:21.826Z 30 Acked 1029 Write 8 Sent Sent Sent false
105352024-02-08T06:43:21.826Z STATES DS:0 DS:1 DS:2 TOTAL
105362024-02-08T06:43:21.826Z New 0 0 0 0
105372024-02-08T06:43:21.826Z Sent 1 1 18 20
105382024-02-08T06:43:21.826Z Done 19 19 2 40
105392024-02-08T06:43:21.826Z Skipped 0 0 0 0
105402024-02-08T06:43:21.826Z Error 0 0 0 0
105412024-02-08T06:43:21.826ZLast Flush: 1009 1009 1009
105422024-02-08T06:43:21.827ZDownstairs last five completed: 1009 1008 1007 1006 1005
105432024-02-08T06:43:21.827ZUpstairs last five completed: 30 29 28 27 26
105442024-02-08T06:43:21.827Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:21.054911765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1522}
105452024-02-08T06:43:21.827ZWrote out file "/tmp/test_repair_verify.data"
105462024-02-08T06:43:21.830Z
105472024-02-08T06:43:21.830ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
105482024-02-08T06:43:21.849ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105492024-02-08T06:43:21.849Z 0 000-019 95 95 95 153 153 153 F F F
105502024-02-08T06:43:21.849Z 1 020-039 91 91 91 145 145 145 T T F <---
105512024-02-08T06:43:21.849Z 2 040-059 93 93 93 148 148 148 F F F
105522024-02-08T06:43:21.849Z 3 060-079 93 93 93 148 148 148 F F F
105532024-02-08T06:43:21.849Z 4 080-099 96 96 96 155 155 155 F F F
105542024-02-08T06:43:21.853Z 5 100-119 96 96 96 155 155 155 T T T
105552024-02-08T06:43:21.853Z 6 120-139 93 93 93 148 148 148 T T T
105562024-02-08T06:43:21.853Z 7 140-159 93 93 93 148 148 148 T T F <---
105572024-02-08T06:43:21.853Z 8 160-179 92 92 92 147 147 147 T T F <---
105582024-02-08T06:43:21.853Z 9 180-199 89 89 89 140 140 140 F F F
105592024-02-08T06:43:21.853Z 10 200-219 94 94 94 149 149 149 T T F <---
105602024-02-08T06:43:21.853Z 11 220-239 94 94 94 152 152 152 F F F
105612024-02-08T06:43:21.853Z 12 240-259 85 85 85 130 130 130 F F F
105622024-02-08T06:43:21.854Z 13 260-279 96 96 96 155 155 155 T T F <---
105632024-02-08T06:43:21.854Z 14 280-299 95 95 95 153 153 153 T T F <---
105642024-02-08T06:43:21.854Z 15 300-319 95 95 95 153 153 153 T T F <---
105652024-02-08T06:43:21.854Z 16 320-339 93 93 93 148 148 148 F F F
105662024-02-08T06:43:21.854Z 17 340-359 80 80 80 123 123 123 F F F
105672024-02-08T06:43:21.854Z 18 360-379 94 94 94 150 150 150 F F F
105682024-02-08T06:43:21.854Z 19 380-399 94 94 94 149 149 149 T T F <---
105692024-02-08T06:43:21.854Z 20 400-419 96 96 96 155 155 155 T T F <---
105702024-02-08T06:43:21.854Z 21 420-439 96 96 96 155 155 155 F F F
105712024-02-08T06:43:21.854Z 22 440-459 90 90 90 144 144 144 F F F
105722024-02-08T06:43:21.854Z 23 460-479 93 93 93 148 148 148 T T F <---
105732024-02-08T06:43:21.854Z 24 480-499 93 93 93 148 148 148 F F F
105742024-02-08T06:43:21.854Z 25 500-519 93 93 93 148 148 148 T T F <---
105752024-02-08T06:43:21.854Z 26 520-539 94 94 94 151 151 151 F F F
105762024-02-08T06:43:21.854Z 27 540-559 92 92 92 147 147 147 F F F
105772024-02-08T06:43:21.855Z 28 560-579 91 91 91 145 145 145 F F F
105782024-02-08T06:43:21.855Z 29 580-599 94 94 94 149 149 149 T T T
105792024-02-08T06:43:21.855ZMax gen: 96, Max flush: 155
105802024-02-08T06:43:21.855ZError: Difference in extent metadata found!
105812024-02-08T06:43:21.855ZA Difference in extent metadata is expected here
105822024-02-08T06:43:21.855ZOn loop 48
105832024-02-08T06:43:21.855Z
105842024-02-08T06:43:21.858ZVerifying data now
105852024-02-08T06:43:27.558ZNo change after verify
105862024-02-08T06:43:27.558ZLoop: 48 Downstairs dump after verify (and repair):
105872024-02-08T06:43:27.573ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105882024-02-08T06:43:27.573Z 0 000-019 95 95 95 153 153 153 F F F
105892024-02-08T06:43:27.573Z 1 020-039 97 97 97 156 156 156 F F F
105902024-02-08T06:43:27.573Z 2 040-059 93 93 93 148 148 148 F F F
105912024-02-08T06:43:27.573Z 3 060-079 93 93 93 148 148 148 F F F
105922024-02-08T06:43:27.573Z 4 080-099 96 96 96 155 155 155 F F F
105932024-02-08T06:43:27.577Z 5 100-119 97 97 97 156 156 156 F F F
105942024-02-08T06:43:27.577Z 6 120-139 97 97 97 156 156 156 F F F
105952024-02-08T06:43:27.577Z 7 140-159 97 97 97 156 156 156 F F F
105962024-02-08T06:43:27.577Z 8 160-179 97 97 97 156 156 156 F F F
105972024-02-08T06:43:27.577Z 9 180-199 89 89 89 140 140 140 F F F
105982024-02-08T06:43:27.577Z 10 200-219 97 97 97 156 156 156 F F F
105992024-02-08T06:43:27.577Z 11 220-239 94 94 94 152 152 152 F F F
106002024-02-08T06:43:27.577Z 12 240-259 85 85 85 130 130 130 F F F
106012024-02-08T06:43:27.577Z 13 260-279 97 97 97 156 156 156 F F F
106022024-02-08T06:43:27.577Z 14 280-299 97 97 97 156 156 156 F F F
106032024-02-08T06:43:27.578Z 15 300-319 97 97 97 156 156 156 F F F
106042024-02-08T06:43:27.578Z 16 320-339 93 93 93 148 148 148 F F F
106052024-02-08T06:43:27.578Z 17 340-359 80 80 80 123 123 123 F F F
106062024-02-08T06:43:27.578Z 18 360-379 94 94 94 150 150 150 F F F
106072024-02-08T06:43:27.578Z 19 380-399 97 97 97 156 156 156 F F F
106082024-02-08T06:43:27.578Z 20 400-419 97 97 97 156 156 156 F F F
106092024-02-08T06:43:27.578Z 21 420-439 96 96 96 155 155 155 F F F
106102024-02-08T06:43:27.578Z 22 440-459 90 90 90 144 144 144 F F F
106112024-02-08T06:43:27.578Z 23 460-479 97 97 97 156 156 156 F F F
106122024-02-08T06:43:27.578Z 24 480-499 93 93 93 148 148 148 F F F
106132024-02-08T06:43:27.578Z 25 500-519 97 97 97 156 156 156 F F F
106142024-02-08T06:43:27.578Z 26 520-539 94 94 94 151 151 151 F F F
106152024-02-08T06:43:27.578Z 27 540-559 92 92 92 147 147 147 F F F
106162024-02-08T06:43:27.578Z 28 560-579 91 91 91 145 145 145 F F F
106172024-02-08T06:43:27.578Z 29 580-599 97 97 97 156 156 156 F F F
106182024-02-08T06:43:27.579ZMax gen: 97, Max flush: 156
106192024-02-08T06:43:27.579Z
106202024-02-08T06:43:27.579ZBegin loop 49
106212024-02-08T06:43:27.579ZDownstairs to restart: 0
106222024-02-08T06:43:27.594Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826276812Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530}
106232024-02-08T06:43:27.594ZCrucible runtime is spawned
106242024-02-08T06:43:27.594Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826606528Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530}
106252024-02-08T06:43:27.594Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826637895Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530}
106262024-02-08T06:43:27.599Z{"msg":"Crucible 69a06c45-dbec-4407-afa1-b2838a75df3e has session id: 84601bc3-ee67-4ea3-8678-5dbd302bc628","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826657744Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106272024-02-08T06:43:27.599Z{"msg":"Upstairs opts: Upstairs UUID: 69a06c45-dbec-4407-afa1-b2838a75df3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826696591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106282024-02-08T06:43:27.599Z{"msg":"Crucible stats registered with UUID: 69a06c45-dbec-4407-afa1-b2838a75df3e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826721899Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106292024-02-08T06:43:27.600Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826756166Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530}
106302024-02-08T06:43:27.600Z{"msg":"69a06c45-dbec-4407-afa1-b2838a75df3e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826778295Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106312024-02-08T06:43:27.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826804443Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106322024-02-08T06:43:27.600Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.82683667Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106332024-02-08T06:43:27.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826883027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106342024-02-08T06:43:27.600Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826911995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106352024-02-08T06:43:27.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826936163Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106362024-02-08T06:43:27.600Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826960651Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106372024-02-08T06:43:27.600Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.826997438Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"io task","client":"0","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106382024-02-08T06:43:27.600Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.827025066Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"io task","client":"2","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106392024-02-08T06:43:27.600Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.827059383Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106402024-02-08T06:43:27.600Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.827096191Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"io task","client":"0","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106412024-02-08T06:43:27.600Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.827126429Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106422024-02-08T06:43:27.600Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.827153707Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"io task","client":"2","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106432024-02-08T06:43:27.600Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.82777614Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106442024-02-08T06:43:27.601Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.827830036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106452024-02-08T06:43:27.601Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.827880152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106462024-02-08T06:43:27.601Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.827920239Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106472024-02-08T06:43:27.601Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.828469058Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106482024-02-08T06:43:27.601Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.828531373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106492024-02-08T06:43:27.601Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.828870378Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106502024-02-08T06:43:27.601Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.828926014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106512024-02-08T06:43:27.601Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.828949942Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106522024-02-08T06:43:27.601Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.829030406Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106532024-02-08T06:43:27.601Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.829060393Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106542024-02-08T06:43:27.601Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.829089651Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106552024-02-08T06:43:27.601Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.82910895Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106562024-02-08T06:43:27.601Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.829135808Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106572024-02-08T06:43:27.601Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.829624911Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106582024-02-08T06:43:27.601Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830083607Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106592024-02-08T06:43:27.602Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830131253Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106602024-02-08T06:43:27.602Z{"msg":"[0]R flush_numbers[0..12]: [153, 156, 148, 148, 155, 156, 156, 156, 156, 140, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.83016794Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106612024-02-08T06:43:27.606Z{"msg":"[0]R generation[0..12]: [95, 97, 93, 93, 96, 97, 97, 97, 97, 89, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830203798Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106622024-02-08T06:43:27.606Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830242025Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106632024-02-08T06:43:27.606Z{"msg":"[1]R flush_numbers[0..12]: [153, 156, 148, 148, 155, 156, 156, 156, 156, 140, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830268273Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106642024-02-08T06:43:27.606Z{"msg":"[1]R generation[0..12]: [95, 97, 93, 93, 96, 97, 97, 97, 97, 89, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.83030273Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106652024-02-08T06:43:27.606Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830338677Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
106662024-02-08T06:43:27.606Z84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106672024-02-08T06:43:27.606Z{"msg":"[2]R flush_numbers[0..12]: [153, 156, 148, 148, 155, 156, 156, 156, 156, 140, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830369915Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106682024-02-08T06:43:27.606Z{"msg":"[2]R generation[0..12]: [95, 97, 93, 93, 96, 97, 97, 97, 97, 89, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830408852Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106692024-02-08T06:43:27.607Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830450779Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106702024-02-08T06:43:27.607Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830489776Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106712024-02-08T06:43:27.607Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830518014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106722024-02-08T06:43:27.607Z{"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830544202Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106732024-02-08T06:43:27.607Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830581299Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106742024-02-08T06:43:27.607Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830618206Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106752024-02-08T06:43:27.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830643794Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106762024-02-08T06:43:27.607ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
106772024-02-08T06:43:27.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830672882Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106782024-02-08T06:43:27.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.83070976Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106792024-02-08T06:43:27.607Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830729678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106802024-02-08T06:43:27.607Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830751466Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106812024-02-08T06:43:27.607Z{"msg":"69a06c45-dbec-4407-afa1-b2838a75df3e is now active with session: 84601bc3-ee67-4ea3-8678-5dbd302bc628","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830765215Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106822024-02-08T06:43:27.607Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830778224Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530,"session_id":"84601bc3-ee67-4ea3-8678-5dbd302bc628"}
106832024-02-08T06:43:27.607Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:26.830790783Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530}
106842024-02-08T06:43:27.608ZLoading write count information from file "/tmp/test_repair_verify.data"
106852024-02-08T06:43:27.608ZRun Repair workload
106862024-02-08T06:43:27.608Z01/30 Write block 87 len 8192 data: 8 8
106872024-02-08T06:43:27.608Z02/30 Read block 435 len 28672
106882024-02-08T06:43:27.608Z03/30 Write block 524 len 4096 data: 6
106892024-02-08T06:43:27.608Z04/30 Read block 445 len 28672
106902024-02-08T06:43:27.608Z05/30 Write block 279 len 4096 data: 7
106912024-02-08T06:43:27.608Z06/30 Flush
106922024-02-08T06:43:27.608Z07/30 Write block 120 len 24576 data: 4 3 3 3 5 6
106932024-02-08T06:43:27.611Z08/30 Write block 161 len 40960 data: 8 10 10 11 12 13 13 10 11 10
106942024-02-08T06:43:27.611Z09/30 Read block 121 len 20480
106952024-02-08T06:43:27.611Z10/30 Read block 65 len 40960
106962024-02-08T06:43:27.611Z11/30 Read block 352 len 8192
106972024-02-08T06:43:27.611Z12/30 Read block 504 len 12288
106982024-02-08T06:43:27.612Z13/30 Write block 250 len 12288 data: 11 11 8
106992024-02-08T06:43:27.612Z14/30 Read block 476 len 24576
107002024-02-08T06:43:27.614Z15/30 Write block 389 len 20480 data: 10 7 7 8 5
107012024-02-08T06:43:27.615Z16/30 Flush
107022024-02-08T06:43:27.617Z17/30 Write block 339 len 36864 data: 12 11 10 9 8 8 9 9 7
107032024-02-08T06:43:27.617Z18/30 Write block 103 len 8192 data: 11 12
107042024-02-08T06:43:27.617Z19/30 Flush
107052024-02-08T06:43:27.620Z20/30 Write block 147 len 12288 data: 4 5 7
107062024-02-08T06:43:27.620Z21/30 Read block 456 len 12288
107072024-02-08T06:43:27.623Z22/30 Write block 386 len 4096 data: 8
107082024-02-08T06:43:27.624Z23/30 Read block 196 len 20480
107092024-02-08T06:43:27.624Z24/30 Read block 15 len 16384
107102024-02-08T06:43:27.624Z25/30 Write block 167 len 36864 data: 14 11 12 11 8 9 11 8 9
107112024-02-08T06:43:27.627Z26/30 Read block 330 len 36864
107122024-02-08T06:43:27.627Z27/30 Read block 311 len 16384
107132024-02-08T06:43:27.627Z28/30 Write block 176 len 28672 data: 8 8 6 6 7 8 10
107142024-02-08T06:43:27.627Z29/30 Write block 153 len 16384 data: 7 8 8 9
107152024-02-08T06:43:27.633Z30/30 Write block 577 len 36864 data: 8 8 10 8 7 8 8 8 7
107162024-02-08T06:43:27.633Z----------------------------------------------------------------
107172024-02-08T06:43:27.633Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
107182024-02-08T06:43:27.633ZStates: Active Active Active
107192024-02-08T06:43:27.633ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
107202024-02-08T06:43:27.633Z 1 Acked 1000 Write 2 Done Done Done false
107212024-02-08T06:43:27.633Z 2 Acked 1001 Read 7 Done Done Done false
107222024-02-08T06:43:27.633Z 3 Acked 1002 Write 1 Done Done Done false
107232024-02-08T06:43:27.633Z 4 Acked 1003 Read 7 Done Done Done false
107242024-02-08T06:43:27.633Z 5 Acked 1004 Write 1 Sent Done Done false
107252024-02-08T06:43:27.634Z 6 Acked 1005 Flush 0 Sent Done Done false
107262024-02-08T06:43:27.634Z 7 Acked 1006 Write 6 Sent Done Done false
107272024-02-08T06:43:27.634Z 8 Acked 1007 Write 10 Sent Done Done false
107282024-02-08T06:43:27.634Z 9 Acked 1008 Read 5 Sent Done Done false
107292024-02-08T06:43:27.634Z 10 Acked 1009 Read 10 Sent Done Done false
107302024-02-08T06:43:27.634Z 11 Acked 1010 Read 2 Sent Done Done false
107312024-02-08T06:43:27.634Z 12 Acked 1011 Read 3 Sent Done Done false
107322024-02-08T06:43:27.634Z 13 Acked 1012 Write 3 Sent Done Done false
107332024-02-08T06:43:27.634Z 14 Acked 1013 Read 6 Sent Done Done false
107342024-02-08T06:43:27.634Z 15 Acked 1014 Write 5 Sent Done Done false
107352024-02-08T06:43:27.634Z 16 Acked 1015 Flush 0 Sent Done Done false
107362024-02-08T06:43:27.634Z 17 Acked 1016 Write 9 Sent Done Done false
107372024-02-08T06:43:27.634Z 18 Acked 1017 Write 2 Sent Done Done false
107382024-02-08T06:43:27.634Z 19 Acked 1018 Flush 0 Sent Done Done false
107392024-02-08T06:43:27.635Z 20 Acked 1019 Write 3 Sent Done Done false
107402024-02-08T06:43:27.635Z 21 Acked 1020 Read 3 Sent Done Done false
107412024-02-08T06:43:27.635Z 22 Acked 1021 Write 1 Sent Done Done false
107422024-02-08T06:43:27.635Z 23 Acked 1022 Read 5 Sent Done Done false
107432024-02-08T06:43:27.635Z 24 Acked 1023 Read 4 Sent Done Done false
107442024-02-08T06:43:27.635Z 25 Acked 1024 Write 9 Sent Done Done false
107452024-02-08T06:43:27.635Z 26 Acked 1025 Read 9 Sent Done Done false
107462024-02-08T06:43:27.635Z 27 Acked 1026 Read 4 Sent Done Done false
107472024-02-08T06:43:27.635Z 28 Acked 1027 Write 7 Sent Done Done false
107482024-02-08T06:43:27.635Z 29 Acked 1028 Write 4 Sent Done Done false
107492024-02-08T06:43:27.635Z 30 Acked 1029 Write 9 Sent Sent Sent false
107502024-02-08T06:43:27.635Z STATES DS:0 DS:1 DS:2 TOTAL
107512024-02-08T06:43:27.635Z New 0 0 0 0
107522024-02-08T06:43:27.635Z Sent 26 1 1 28
107532024-02-08T06:43:27.635Z Done 4 29 29 62
107542024-02-08T06:43:27.635Z Skipped 0 0 0 0
107552024-02-08T06:43:27.636Z Error 0 0 0 0
107562024-02-08T06:43:27.636ZLast Flush: 0 1018 1018
107572024-02-08T06:43:27.636ZDownstairs last five completed:
107582024-02-08T06:43:27.636ZUpstairs last five completed: 30 29 28 27 26
107592024-02-08T06:43:27.636Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:26.861169153Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1530}
107602024-02-08T06:43:27.636ZWrote out file "/tmp/test_repair_verify.data"
107612024-02-08T06:43:27.639Z
107622024-02-08T06:43:27.639ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
107632024-02-08T06:43:27.653ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107642024-02-08T06:43:27.653Z 0 000-019 95 95 95 153 153 153 F F F
107652024-02-08T06:43:27.653Z 1 020-039 97 97 97 156 156 156 F F F
107662024-02-08T06:43:27.653Z 2 040-059 93 93 93 148 148 148 F F F
107672024-02-08T06:43:27.653Z 3 060-079 93 93 93 148 148 148 F F F
107682024-02-08T06:43:27.653Z 4 080-099 96 98 98 155 157 157 T F F <---
107692024-02-08T06:43:27.653Z 5 100-119 97 98 98 156 159 159 F F F <---
107702024-02-08T06:43:27.653Z 6 120-139 97 98 98 156 158 158 F F F <---
107712024-02-08T06:43:27.653Z 7 140-159 97 97 97 156 156 156 F T T <---
107722024-02-08T06:43:27.653Z 8 160-179 97 98 98 156 158 158 F T T <---
107732024-02-08T06:43:27.653Z 9 180-199 89 89 89 140 140 140 F T T <---
107742024-02-08T06:43:27.654Z 10 200-219 97 97 97 156 156 156 F F F
107752024-02-08T06:43:27.657Z 11 220-239 94 94 94 152 152 152 F F F
107762024-02-08T06:43:27.657Z 12 240-259 85 98 98 130 158 158 F F F <---
107772024-02-08T06:43:27.657Z 13 260-279 97 98 98 156 157 157 F F F <---
107782024-02-08T06:43:27.657Z 14 280-299 97 97 97 156 156 156 F F F
107792024-02-08T06:43:27.657Z 15 300-319 97 97 97 156 156 156 F F F
107802024-02-08T06:43:27.657Z 16 320-339 93 98 98 148 159 159 F F F <---
107812024-02-08T06:43:27.657Z 17 340-359 80 98 98 123 159 159 F F F <---
107822024-02-08T06:43:27.658Z 18 360-379 94 94 94 150 150 150 F F F
107832024-02-08T06:43:27.658Z 19 380-399 97 98 98 156 158 158 F T T <---
107842024-02-08T06:43:27.658Z 20 400-419 97 97 97 156 156 156 F F F
107852024-02-08T06:43:27.658Z 21 420-439 96 96 96 155 155 155 F F F
107862024-02-08T06:43:27.658Z 22 440-459 90 90 90 144 144 144 F F F
107872024-02-08T06:43:27.658Z 23 460-479 97 97 97 156 156 156 F F F
107882024-02-08T06:43:27.658Z 24 480-499 93 93 93 148 148 148 F F F
107892024-02-08T06:43:27.658Z 25 500-519 97 97 97 156 156 156 F F F
107902024-02-08T06:43:27.658Z 26 520-539 94 98 98 151 157 157 T F F <---
107912024-02-08T06:43:27.658Z 27 540-559 92 92 92 147 147 147 F F F
107922024-02-08T06:43:27.658Z 28 560-579 91 91 91 145 145 145 F T T <---
107932024-02-08T06:43:27.658Z 29 580-599 97 97 97 156 156 156 F T T <---
107942024-02-08T06:43:27.658ZMax gen: 98, Max flush: 159
107952024-02-08T06:43:27.658ZError: Difference in extent metadata found!
107962024-02-08T06:43:27.658ZA Difference in extent metadata is expected here
107972024-02-08T06:43:27.658ZOn loop 49
107982024-02-08T06:43:27.659Z
107992024-02-08T06:43:27.663ZVerifying data now
108002024-02-08T06:43:33.355ZNo change after verify
108012024-02-08T06:43:33.355ZLoop: 49 Downstairs dump after verify (and repair):
108022024-02-08T06:43:33.371ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108032024-02-08T06:43:33.371Z 0 000-019 95 95 95 153 153 153 F F F
108042024-02-08T06:43:33.371Z 1 020-039 97 97 97 156 156 156 F F F
108052024-02-08T06:43:33.371Z 2 040-059 93 93 93 148 148 148 F F F
108062024-02-08T06:43:33.371Z 3 060-079 93 93 93 148 148 148 F F F
108072024-02-08T06:43:33.371Z 4 080-099 98 98 98 157 157 157 F F F
108082024-02-08T06:43:33.371Z 5 100-119 98 98 98 159 159 159 F F F
108092024-02-08T06:43:33.375Z 6 120-139 98 98 98 158 158 158 F F F
108102024-02-08T06:43:33.375Z 7 140-159 99 99 99 160 160 160 F F F
108112024-02-08T06:43:33.375Z 8 160-179 99 99 99 160 160 160 F F F
108122024-02-08T06:43:33.375Z 9 180-199 99 99 99 160 160 160 F F F
108132024-02-08T06:43:33.375Z 10 200-219 97 97 97 156 156 156 F F F
108142024-02-08T06:43:33.375Z 11 220-239 94 94 94 152 152 152 F F F
108152024-02-08T06:43:33.375Z 12 240-259 98 98 98 158 158 158 F F F
108162024-02-08T06:43:33.375Z 13 260-279 98 98 98 157 157 157 F F F
108172024-02-08T06:43:33.375Z 14 280-299 97 97 97 156 156 156 F F F
108182024-02-08T06:43:33.375Z 15 300-319 97 97 97 156 156 156 F F F
108192024-02-08T06:43:33.375Z 16 320-339 98 98 98 159 159 159 F F F
108202024-02-08T06:43:33.375Z 17 340-359 98 98 98 159 159 159 F F F
108212024-02-08T06:43:33.375Z 18 360-379 94 94 94 150 150 150 F F F
108222024-02-08T06:43:33.375Z 19 380-399 99 99 99 160 160 160 F F F
108232024-02-08T06:43:33.376Z 20 400-419 97 97 97 156 156 156 F F F
108242024-02-08T06:43:33.376Z 21 420-439 96 96 96 155 155 155 F F F
108252024-02-08T06:43:33.376Z 22 440-459 90 90 90 144 144 144 F F F
108262024-02-08T06:43:33.376Z 23 460-479 97 97 97 156 156 156 F F F
108272024-02-08T06:43:33.376Z 24 480-499 93 93 93 148 148 148 F F F
108282024-02-08T06:43:33.376Z 25 500-519 97 97 97 156 156 156 F F F
108292024-02-08T06:43:33.376Z 26 520-539 98 98 98 157 157 157 F F F
108302024-02-08T06:43:33.376Z 27 540-559 92 92 92 147 147 147 F F F
108312024-02-08T06:43:33.376Z 28 560-579 99 99 99 160 160 160 F F F
108322024-02-08T06:43:33.376Z 29 580-599 99 99 99 160 160 160 F F F
108332024-02-08T06:43:33.376ZMax gen: 99, Max flush: 160
108342024-02-08T06:43:33.376Z
108352024-02-08T06:43:33.376ZBegin loop 50
108362024-02-08T06:43:33.376ZDownstairs to restart: 2
108372024-02-08T06:43:33.395Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.626385822Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538}
108382024-02-08T06:43:33.395Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
108392024-02-08T06:43:33.395Z2024-02-08T06:43:32.626669971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538}
108402024-02-08T06:43:33.400Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.626758234Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538}
108412024-02-08T06:43:33.400Z{"msg":"Crucible 62c99a0b-81f2-4305-99b0-16e97048158c has session id: 394685cc-9c41-4a29-9af1-d49503fdb4b8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.626797202Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108422024-02-08T06:43:33.400Z{"msg":"Upstairs opts: Upstairs UUID: 62c99a0b-81f2-4305-99b0-16e97048158c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.62681695Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108432024-02-08T06:43:33.401Z{"msg":"Crucible stats registered with UUID: 62c99a0b-81f2-4305-99b0-16e97048158c","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.626835099Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108442024-02-08T06:43:33.401Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.626856597Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538}
108452024-02-08T06:43:33.401Z{"msg":"62c99a0b-81f2-4305-99b0-16e97048158c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.626915943Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108462024-02-08T06:43:33.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.626941521Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108472024-02-08T06:43:33.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.626961579Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108482024-02-08T06:43:33.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.626988527Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108492024-02-08T06:43:33.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.627023894Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108502024-02-08T06:43:33.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.627047903Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108512024-02-08T06:43:33.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.627071721Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108522024-02-08T06:43:33.401Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.627166714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"io task","client":"0","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108532024-02-08T06:43:33.401Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.627235909Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"io task","client":"1","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108542024-02-08T06:43:33.401Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.627267176Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108552024-02-08T06:43:33.401Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.627293114Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108562024-02-08T06:43:33.401Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.627318012Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"io task","client":"1","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108572024-02-08T06:43:33.401Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.62735592Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"io task","client":"0","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108582024-02-08T06:43:33.402Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.627993012Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108592024-02-08T06:43:33.402Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.62802125Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108602024-02-08T06:43:33.402Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.628036629Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108612024-02-08T06:43:33.402Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.628066546Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108622024-02-08T06:43:33.402Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.628589667Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108632024-02-08T06:43:33.402Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.628718177Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108642024-02-08T06:43:33.402Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.62895399Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108652024-02-08T06:43:33.402Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.629028184Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108662024-02-08T06:43:33.402Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.629104688Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108672024-02-08T06:43:33.402Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.629132936Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108682024-02-08T06:43:33.402Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.629180313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108692024-02-08T06:43:33.402Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.629267356Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108702024-02-08T06:43:33.402Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.629299154Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108712024-02-08T06:43:33.402Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.629314033Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108722024-02-08T06:43:33.402Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.629605591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108732024-02-08T06:43:33.403Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630084605Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108742024-02-08T06:43:33.403Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630110203Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108752024-02-08T06:43:33.403Z{"msg":"[0]R flush_numbers[0..12]: [153, 156, 148, 148, 157, 159, 158, 160, 160, 160, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630126222Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108762024-02-08T06:43:33.403Z{"msg":"[0]R generation[0..12]: [95, 97, 93, 93, 98, 98, 98, 99, 99, 99, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.63014709Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108772024-02-08T06:43:33.403Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630161829Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108782024-02-08T06:43:33.403Z{"msg":"[1]R flush_numbers[0..12]: [153, 156, 148, 148, 157, 159, 158, 160, 160, 160, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630182847Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108792024-02-08T06:43:33.403Z{"msg":"[1]R generation[0..12]: [95, 97, 93, 93, 98, 98, 98, 99, 99, 99, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630196976Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108802024-02-08T06:43:33.403Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630217445Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108812024-02-08T06:43:33.403Z{"msg":"[2]R flush_numbers[0..12]: [153, 156, 148, 148, 157, 159, 158, 160, 160, 160, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630231824Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108822024-02-08T06:43:33.403Z{"msg":"[2]R generation[0..12]: [95, 97, 93, 93, 98, 98, 98, 99, 99, 99, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630246283Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108832024-02-08T06:43:33.403Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630267671Z","Wait for a query_work_queue command to finish before sending IO
108842024-02-08T06:43:33.403Zhostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108852024-02-08T06:43:33.407Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.63028414Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108862024-02-08T06:43:33.407Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630305178Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108872024-02-08T06:43:33.407Z{"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630318817Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108882024-02-08T06:43:33.407Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630332876Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108892024-02-08T06:43:33.407Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630347585Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108902024-02-08T06:43:33.407Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630359664Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108912024-02-08T06:43:33.407Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630380653Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108922024-02-08T06:43:33.407Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630401491Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108932024-02-08T06:43:33.407Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630426019Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108942024-02-08T06:43:33.408Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630447948Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108952024-02-08T06:43:33.408Z{"msg":"62c99a0b-81f2-4305-99b0-16e97048158c is now active with session: 394685cc-9c41-4a29-9af1-d49503fdb4b8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630478985Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108962024-02-08T06:43:33.408Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630501664Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538,"session_id":"394685cc-9c41-4a29-9af1-d49503fdb4b8"}
108972024-02-08T06:43:33.408Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:32.630528482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538}
108982024-02-08T06:43:33.408ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108992024-02-08T06:43:33.408ZLoading write count information from file "/tmp/test_repair_verify.data"
109002024-02-08T06:43:33.408ZRun Repair workload
109012024-02-08T06:43:33.408Z01/30 Write block 50 len 28672 data: 5 7 7 7 9 9 11
109022024-02-08T06:43:33.408Z02/30 Read block 54 len 4096
109032024-02-08T06:43:33.408Z03/30 Write block 118 len 28672 data: 5 4 5 4 4 4 6
109042024-02-08T06:43:33.408Z04/30 Read block 208 len 4096
109052024-02-08T06:43:33.408Z05/30 Write block 490 len 16384 data: 8 9 8 8
109062024-02-08T06:43:33.408Z06/30 Write block 202 len 28672 data: 10 8 8 9 9 10 7
109072024-02-08T06:43:33.408Z07/30 Write block 527 len 8192 data: 7 8
109082024-02-08T06:43:33.408Z08/30 Read block 110 len 24576
109092024-02-08T06:43:33.409Z09/30 Read block 269 len 40960
109102024-02-08T06:43:33.412Z10/30 Read block 112 len 36864
109112024-02-08T06:43:33.412Z11/30 Write block 24 len 36864 data: 11 12 11 11 11 10 10 9 8
109122024-02-08T06:43:33.412Z12/30 Read block 470 len 16384
109132024-02-08T06:43:33.412Z13/30 Read block 123 len 40960
109142024-02-08T06:43:33.415Z14/30 Read block 228 len 32768
109152024-02-08T06:43:33.415Z15/30 Read block 530 len 28672
109162024-02-08T06:43:33.415Z16/30 Write block 417 len 4096 data: 7
109172024-02-08T06:43:33.415Z17/30 Write block 34 len 32768 data: 8 8 7 6 8 5 5 5
109182024-02-08T06:43:33.418Z18/30 Read block 316 len 32768
109192024-02-08T06:43:33.418Z19/30 Write block 489 len 20480 data: 9 9 10 9 9
109202024-02-08T06:43:33.418Z20/30 Read block 554 len 8192
109212024-02-08T06:43:33.418Z21/30 Flush
109222024-02-08T06:43:33.421Z22/30 Flush
109232024-02-08T06:43:33.424Z23/30 Write block 278 len 28672 data: 5 8 6 6 6 7 5
109242024-02-08T06:43:33.425Z24/30 Read block 380 len 12288
109252024-02-08T06:43:33.425Z25/30 Read block 431 len 8192
109262024-02-08T06:43:33.425Z26/30 Write block 459 len 32768 data: 10 7 8 9 9 7 6 7
109272024-02-08T06:43:33.425Z27/30 Read block 578 len 24576
109282024-02-08T06:43:33.431Z28/30 Read block 466 len 20480
109292024-02-08T06:43:33.431Z29/30 Read block 566 len 28672
109302024-02-08T06:43:33.431Z30/30 Read block 254 len 36864
109312024-02-08T06:43:33.434Z----------------------------------------------------------------
109322024-02-08T06:43:33.434Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
109332024-02-08T06:43:33.434ZStates: Active Active Active
109342024-02-08T06:43:33.434ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
109352024-02-08T06:43:33.434Z 1 Acked 1000 Write 7 Done Done Done false
109362024-02-08T06:43:33.434Z 2 Acked 1001 Read 1 Done Done Done false
109372024-02-08T06:43:33.434Z 3 Acked 1002 Write 7 Done Done Sent false
109382024-02-08T06:43:33.435Z 4 Acked 1003 Read 1 Done Done Sent false
109392024-02-08T06:43:33.435Z 5 Acked 1004 Write 4 Done Done Sent false
109402024-02-08T06:43:33.435Z 6 Acked 1005 Write 7 Done Done Sent false
109412024-02-08T06:43:33.435Z 7 Acked 1006 Write 2 Done Done Sent false
109422024-02-08T06:43:33.435Z 8 Acked 1007 Read 6 Done Done Sent false
109432024-02-08T06:43:33.435Z 9 Acked 1008 Read 10 Done Done Sent false
109442024-02-08T06:43:33.435Z 10 Acked 1009 Read 9 Done Done Sent false
109452024-02-08T06:43:33.435Z 11 Acked 1010 Write 9 Done Done Sent false
109462024-02-08T06:43:33.435Z 12 Acked 1011 Read 4 Done Done Sent false
109472024-02-08T06:43:33.435Z 13 Acked 1012 Read 10 Done Done Sent false
109482024-02-08T06:43:33.435Z 14 Acked 1013 Read 8 Done Done Sent false
109492024-02-08T06:43:33.435Z 15 Acked 1014 Read 7 Done Done Sent false
109502024-02-08T06:43:33.435Z 16 Acked 1015 Write 1 Done Done Sent false
109512024-02-08T06:43:33.435Z 17 Acked 1016 Write 8 Done Done Sent false
109522024-02-08T06:43:33.435Z 18 Acked 1017 Read 8 Done Done Sent false
109532024-02-08T06:43:33.435Z 19 Acked 1018 Write 5 Done Done Sent false
109542024-02-08T06:43:33.436Z 20 Acked 1019 Read 2 Done Done Sent false
109552024-02-08T06:43:33.436Z 21 Acked 1020 Flush 0 Done Done Sent false
109562024-02-08T06:43:33.436Z 22 Acked 1021 Flush 0 Done Done Sent false
109572024-02-08T06:43:33.436Z 23 Acked 1022 Write 7 Done Done Sent false
109582024-02-08T06:43:33.436Z 24 Acked 1023 Read 3 Done Done Sent false
109592024-02-08T06:43:33.436Z 25 Acked 1024 Read 2 Done Done Sent false
109602024-02-08T06:43:33.436Z 26 Acked 1025 Write 8 Done Done Sent false
109612024-02-08T06:43:33.436Z 27 Acked 1026 Read 6 Done Done Sent false
109622024-02-08T06:43:33.436Z 28 Acked 1027 Read 5 Done Done Sent false
109632024-02-08T06:43:33.436Z 29 Acked 1028 Read 7 Done Done Sent false
109642024-02-08T06:43:33.436Z 30 Acked 1029 Read 9 Done Sent Sent false
109652024-02-08T06:43:33.436Z STATES DS:0 DS:1 DS:2 TOTAL
109662024-02-08T06:43:33.436Z New 0 0 0 0
109672024-02-08T06:43:33.437Z Sent 0 1 28 29
109682024-02-08T06:43:33.437Z Done 30 29 2 61
109692024-02-08T06:43:33.437Z Skipped 0 0 0 0
109702024-02-08T06:43:33.437Z Error 0 0 0 0
109712024-02-08T06:43:33.437ZLast Flush: 1021 1021 0
109722024-02-08T06:43:33.437ZDownstairs last five completed:
109732024-02-08T06:43:33.437ZUpstairs last five completed: 30 29 28 27 26
109742024-02-08T06:43:33.437Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:32.661267554Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1538}
109752024-02-08T06:43:33.437ZWrote out file "/tmp/test_repair_verify.data"
109762024-02-08T06:43:33.437Z
109772024-02-08T06:43:33.440ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
109782024-02-08T06:43:33.452ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109792024-02-08T06:43:33.452Z 0 000-019 95 95 95 153 153 153 F F F
109802024-02-08T06:43:33.452Z 1 020-039 100 100 97 161 161 156 F F F <---
109812024-02-08T06:43:33.452Z 2 040-059 100 100 93 161 161 148 F F T <---
109822024-02-08T06:43:33.452Z 3 060-079 93 93 93 148 148 148 F F F
109832024-02-08T06:43:33.453Z 4 080-099 98 98 98 157 157 157 F F F
109842024-02-08T06:43:33.453Z 5 100-119 100 100 98 161 161 159 F F F <---
109852024-02-08T06:43:33.453Z 6 120-139 100 100 98 161 161 158 F F F <---
109862024-02-08T06:43:33.453Z 7 140-159 99 99 99 160 160 160 F F F
109872024-02-08T06:43:33.453Z 8 160-179 99 99 99 160 160 160 F F F
109882024-02-08T06:43:33.456Z 9 180-199 99 99 99 160 160 160 F F F
109892024-02-08T06:43:33.456Z 10 200-219 100 100 97 161 161 156 F F F <---
109902024-02-08T06:43:33.456Z 11 220-239 94 94 94 152 152 152 F F F
109912024-02-08T06:43:33.456Z 12 240-259 98 98 98 158 158 158 F F F
109922024-02-08T06:43:33.456Z 13 260-279 98 98 98 157 157 157 T T F <---
109932024-02-08T06:43:33.457Z 14 280-299 97 97 97 156 156 156 T T F <---
109942024-02-08T06:43:33.457Z 15 300-319 97 97 97 156 156 156 F F F
109952024-02-08T06:43:33.457Z 16 320-339 98 98 98 159 159 159 F F F
109962024-02-08T06:43:33.457Z 17 340-359 98 98 98 159 159 159 F F F
109972024-02-08T06:43:33.457Z 18 360-379 94 94 94 150 150 150 F F F
109982024-02-08T06:43:33.457Z 19 380-399 99 99 99 160 160 160 F F F
109992024-02-08T06:43:33.457Z 20 400-419 100 100 97 161 161 156 F F F <---
110002024-02-08T06:43:33.457Z 21 420-439 96 96 96 155 155 155 F F F
110012024-02-08T06:43:33.457Z 22 440-459 90 90 90 144 144 144 T T F <---
110022024-02-08T06:43:33.457Z 23 460-479 97 97 97 156 156 156 T T F <---
110032024-02-08T06:43:33.457Z 24 480-499 100 100 93 161 161 148 F F F <---
110042024-02-08T06:43:33.457Z 25 500-519 97 97 97 156 156 156 F F F
110052024-02-08T06:43:33.457Z 26 520-539 100 100 98 161 161 157 F F F <---
110062024-02-08T06:43:33.457Z 27 540-559 92 92 92 147 147 147 F F F
110072024-02-08T06:43:33.457Z 28 560-579 99 99 99 160 160 160 F F F
110082024-02-08T06:43:33.457Z 29 580-599 99 99 99 160 160 160 F F F
110092024-02-08T06:43:33.458ZMax gen: 100, Max flush: 161
110102024-02-08T06:43:33.458ZError: Difference in extent metadata found!
110112024-02-08T06:43:33.458ZA Difference in extent metadata is expected here
110122024-02-08T06:43:33.458ZOn loop 50
110132024-02-08T06:43:33.458Z
110142024-02-08T06:43:33.461ZVerifying data now
110152024-02-08T06:43:39.065ZNo change after verify
110162024-02-08T06:43:39.066ZLoop: 50 Downstairs dump after verify (and repair):
110172024-02-08T06:43:39.080ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110182024-02-08T06:43:39.080Z 0 000-019 95 95 95 153 153 153 F F F
110192024-02-08T06:43:39.080Z 1 020-039 100 100 100 161 161 161 F F F
110202024-02-08T06:43:39.081Z 2 040-059 100 100 100 161 161 161 F F F
110212024-02-08T06:43:39.081Z 3 060-079 93 93 93 148 148 148 F F F
110222024-02-08T06:43:39.081Z 4 080-099 98 98 98 157 157 157 F F F
110232024-02-08T06:43:39.081Z 5 100-119 100 100 100 161 161 161 F F F
110242024-02-08T06:43:39.081Z 6 120-139 100 100 100 161 161 161 F F F
110252024-02-08T06:43:39.081Z 7 140-159 99 99 99 160 160 160 F F F
110262024-02-08T06:43:39.084Z 8 160-179 99 99 99 160 160 160 F F F
110272024-02-08T06:43:39.084Z 9 180-199 99 99 99 160 160 160 F F F
110282024-02-08T06:43:39.084Z 10 200-219 100 100 100 161 161 161 F F F
110292024-02-08T06:43:39.085Z 11 220-239 94 94 94 152 152 152 F F F
110302024-02-08T06:43:39.085Z 12 240-259 98 98 98 158 158 158 F F F
110312024-02-08T06:43:39.085Z 13 260-279 101 101 101 162 162 162 F F F
110322024-02-08T06:43:39.085Z 14 280-299 101 101 101 162 162 162 F F F
110332024-02-08T06:43:39.085Z 15 300-319 97 97 97 156 156 156 F F F
110342024-02-08T06:43:39.085Z 16 320-339 98 98 98 159 159 159 F F F
110352024-02-08T06:43:39.085Z 17 340-359 98 98 98 159 159 159 F F F
110362024-02-08T06:43:39.085Z 18 360-379 94 94 94 150 150 150 F F F
110372024-02-08T06:43:39.085Z 19 380-399 99 99 99 160 160 160 F F F
110382024-02-08T06:43:39.085Z 20 400-419 100 100 100 161 161 161 F F F
110392024-02-08T06:43:39.085Z 21 420-439 96 96 96 155 155 155 F F F
110402024-02-08T06:43:39.085Z 22 440-459 101 101 101 162 162 162 F F F
110412024-02-08T06:43:39.085Z 23 460-479 101 101 101 162 162 162 F F F
110422024-02-08T06:43:39.085Z 24 480-499 100 100 100 161 161 161 F F F
110432024-02-08T06:43:39.086Z 25 500-519 97 97 97 156 156 156 F F F
110442024-02-08T06:43:39.086Z 26 520-539 100 100 100 161 161 161 F F F
110452024-02-08T06:43:39.086Z 27 540-559 92 92 92 147 147 147 F F F
110462024-02-08T06:43:39.086Z 28 560-579 99 99 99 160 160 160 F F F
110472024-02-08T06:43:39.086Z 29 580-599 99 99 99 160 160 160 F F F
110482024-02-08T06:43:39.086ZMax gen: 101, Max flush: 162
110492024-02-08T06:43:39.086Z
110502024-02-08T06:43:39.086ZBegin loop 51
110512024-02-08T06:43:39.086ZDownstairs to restart: 0
110522024-02-08T06:43:39.104Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.332774519Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546}
110532024-02-08T06:43:39.104Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333037619Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546}
110542024-02-08T06:43:39.111Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333062528Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546}
110552024-02-08T06:43:39.111Z{"msg":"Crucible 5ee2eb98-3a79-4608-9489-80e4eabcd8a0 has session id: 476b6c19-f2c9-4923-975b-137db92a83a2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333096075Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","Crucible runtime is spawned
110562024-02-08T06:43:39.111Zpid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110572024-02-08T06:43:39.111Z{"msg":"Upstairs opts: Upstairs UUID: 5ee2eb98-3a79-4608-9489-80e4eabcd8a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333149071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110582024-02-08T06:43:39.111Z{"msg":"Crucible stats registered with UUID: 5ee2eb98-3a79-4608-9489-80e4eabcd8a0","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333196478Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110592024-02-08T06:43:39.111Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333217906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546}
110602024-02-08T06:43:39.112Z{"msg":"5ee2eb98-3a79-4608-9489-80e4eabcd8a0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333333677Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110612024-02-08T06:43:39.112Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333362555Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110622024-02-08T06:43:39.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333383624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110632024-02-08T06:43:39.112Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333408582Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110642024-02-08T06:43:39.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333435329Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110652024-02-08T06:43:39.112Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333475117Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110662024-02-08T06:43:39.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333502184Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110672024-02-08T06:43:39.112Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333527753Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110682024-02-08T06:43:39.112Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.3335648Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110692024-02-08T06:43:39.112Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333591968Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110702024-02-08T06:43:39.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333623226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110712024-02-08T06:43:39.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333648333Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110722024-02-08T06:43:39.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.333685121Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110732024-02-08T06:43:39.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.334384358Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110742024-02-08T06:43:39.112Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.334425705Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110752024-02-08T06:43:39.113Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.334451573Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110762024-02-08T06:43:39.113Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.33448851Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110772024-02-08T06:43:39.113Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.335069487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110782024-02-08T06:43:39.113Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.335130442Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110792024-02-08T06:43:39.113Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.335488785Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110802024-02-08T06:43:39.113Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.335546231Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110812024-02-08T06:43:39.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.335733027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110822024-02-08T06:43:39.113Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.335770064Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110832024-02-08T06:43:39.113Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.335792372Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110842024-02-08T06:43:39.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.335818141Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110852024-02-08T06:43:39.113Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.335856448Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110862024-02-08T06:43:39.113Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.335900804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110872024-02-08T06:43:39.113Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336084621Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110882024-02-08T06:43:39.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336527137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110892024-02-08T06:43:39.113Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336568984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110902024-02-08T06:43:39.114Z{"msg":"[0]R flush_numbers[0..12]: [153, 161, 161, 148, 157, 161, 161, 160, 160, 160, 161, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336588563Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110912024-02-08T06:43:39.114Z{"msg":"[0]R generation[0..12]: [95, 100, 100, 93, 98, 100, 100, 99, 99, 99, 100, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336611381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110922024-02-08T06:43:39.114Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.33662624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110932024-02-08T06:43:39.114Z{"msg":"[1]R flush_numbers[0..12]: [153, 161, 161, 148, 157, 161, 161, 160, 160, 160, 161, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336647928Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110942024-02-08T06:43:39.114Z{"msg":"[1]R generation[0..12]: [95, 100, 100, 93, 98, 100, 100, 99, 99, 99, 100, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336665197Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110952024-02-08T06:43:39.114Z{"msg":"Wait for a query_work_queue command to finish before sending IO
110962024-02-08T06:43:39.114Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336703214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110972024-02-08T06:43:39.114Z{"msg":"[2]R flush_numbers[0..12]: [153, 161, 161, 148, 157, 161, 161, 160, 160, 160, 161, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336742301Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110982024-02-08T06:43:39.114Z{"msg":"[2]R generation[0..12]: [95, 100, 100, 93, 98, 100, 100, 99, 99, 99, 100, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.33676665Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
110992024-02-08T06:43:39.114Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336781578Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111002024-02-08T06:43:39.114Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336801507Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111012024-02-08T06:43:39.114Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336838544Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111022024-02-08T06:43:39.114Z{"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336869662Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111032024-02-08T06:43:39.114Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336884961Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111042024-02-08T06:43:39.114Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336906899Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111052024-02-08T06:43:39.115Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336941596Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111062024-02-08T06:43:39.115Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.336977074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111072024-02-08T06:43:39.115Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.337009261Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111082024-02-08T06:43:39.115Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.337048468Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111092024-02-08T06:43:39.115Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.337076856Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111102024-02-08T06:43:39.115Z{"msg":"5ee2eb98-3a79-4608-9489-80e4eabcd8a0 is now active with session: 476b6c19-f2c9-4923-975b-137db92a83a2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.337093695Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111112024-02-08T06:43:39.115Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.337114903Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546,"session_id":"476b6c19-f2c9-4923-975b-137db92a83a2"}
111122024-02-08T06:43:39.115Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:38.337146551Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546}
111132024-02-08T06:43:39.115ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
111142024-02-08T06:43:39.115ZLoading write count information from file "/tmp/test_repair_verify.data"
111152024-02-08T06:43:39.115ZRun Repair workload
111162024-02-08T06:43:39.115Z01/30 Write block 500 len 4096 data: 9
111172024-02-08T06:43:39.115Z02/30 Read block 150 len 24576
111182024-02-08T06:43:39.119Z03/30 Read block 212 len 8192
111192024-02-08T06:43:39.119Z04/30 Flush
111202024-02-08T06:43:39.119Z05/30 Flush
111212024-02-08T06:43:39.119Z06/30 Flush
111222024-02-08T06:43:39.119Z07/30 Write block 340 len 16384 data: 12 11 10 9
111232024-02-08T06:43:39.119Z08/30 Read block 279 len 12288
111242024-02-08T06:43:39.119Z09/30 Write block 243 len 12288 data: 9 8 8
111252024-02-08T06:43:39.119Z10/30 Read block 524 len 8192
111262024-02-08T06:43:39.119Z11/30 Read block 43 len 32768
111272024-02-08T06:43:39.119Z12/30 Write block 441 len 20480 data: 7 7 8 6 6
111282024-02-08T06:43:39.119Z13/30 Read block 214 len 12288
111292024-02-08T06:43:39.119Z14/30 Read block 122 len 4096
111302024-02-08T06:43:39.120Z15/30 Read block 425 len 40960
111312024-02-08T06:43:39.124Z16/30 Write block 563 len 40960 data: 7 6 7 7 7 7 6 8 7 7
111322024-02-08T06:43:39.124Z17/30 Write block 264 len 8192 data: 8 9
111332024-02-08T06:43:39.124Z18/30 Flush
111342024-02-08T06:43:39.127Z19/30 Write block 50 len 8192 data: 6 8
111352024-02-08T06:43:39.127Z20/30 Write block 545 len 32768 data: 7 8 12 12 14 12 12 11
111362024-02-08T06:43:39.127Z21/30 Write block 372 len 16384 data: 11 10 4 6
111372024-02-08T06:43:39.127Z22/30 Write block 389 len 4096 data: 11
111382024-02-08T06:43:39.127Z23/30 Write block 359 len 12288 data: 9 7 5
111392024-02-08T06:43:39.127Z24/30 Read block 98 len 20480
111402024-02-08T06:43:39.127Z25/30 Write block 263 len 20480 data: 9 9 10 9 10
111412024-02-08T06:43:39.131Z26/30 Read block 379 len 32768
111422024-02-08T06:43:39.131Z27/30 Write block 277 len 20480 data: 9 6 9 7 7
111432024-02-08T06:43:39.131Z28/30 Read block 155 len 12288
111442024-02-08T06:43:39.131Z29/30 Read block 363 len 8192
111452024-02-08T06:43:39.134Z30/30 Write block 551 len 24576 data: 13 12 11 10 10 14
111462024-02-08T06:43:39.134Z----------------------------------------------------------------
111472024-02-08T06:43:39.134Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
111482024-02-08T06:43:39.134ZStates: Active Active Active
111492024-02-08T06:43:39.135ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
111502024-02-08T06:43:39.135Z 1 Acked 1000 Write 1 Sent Done Done false
111512024-02-08T06:43:39.135Z 2 Acked 1001 Read 6 Sent Done Done false
111522024-02-08T06:43:39.135Z 3 Acked 1002 Read 2 Sent Done Done false
111532024-02-08T06:43:39.135Z 4 Acked 1003 Flush 0 Sent Done Done false
111542024-02-08T06:43:39.135Z 5 Acked 1004 Flush 0 Sent Done Done false
111552024-02-08T06:43:39.135Z 6 Acked 1005 Flush 0 Sent Done Done false
111562024-02-08T06:43:39.135Z 7 Acked 1006 Write 4 Sent Done Done false
111572024-02-08T06:43:39.135Z 8 Acked 1007 Read 3 Sent Done Done false
111582024-02-08T06:43:39.135Z 9 Acked 1008 Write 3 Sent Done Done false
111592024-02-08T06:43:39.135Z 10 Acked 1009 Read 2 Sent Done Done false
111602024-02-08T06:43:39.135Z 11 Acked 1010 Read 8 Sent Done Done false
111612024-02-08T06:43:39.135Z 12 Acked 1011 Write 5 Sent Done Done false
111622024-02-08T06:43:39.135Z 13 Acked 1012 Read 3 Sent Done Done false
111632024-02-08T06:43:39.135Z 14 Acked 1013 Read 1 Sent Done Done false
111642024-02-08T06:43:39.136Z 15 Acked 1014 Read 10 Sent Done Done false
111652024-02-08T06:43:39.136Z 16 Acked 1015 Write 10 Sent Done Done false
111662024-02-08T06:43:39.136Z 17 Acked 1016 Write 2 Sent Done Done false
111672024-02-08T06:43:39.136Z 18 Acked 1017 Flush 0 Sent Done Done false
111682024-02-08T06:43:39.136Z 19 Acked 1018 Write 2 Sent Done Done false
111692024-02-08T06:43:39.136Z 20 Acked 1019 Write 8 Sent Done Done false
111702024-02-08T06:43:39.136Z 21 Acked 1020 Write 4 Sent Done Done false
111712024-02-08T06:43:39.136Z 22 Acked 1021 Write 1 Sent Done Done false
111722024-02-08T06:43:39.136Z 23 Acked 1022 Write 3 Sent Done Done false
111732024-02-08T06:43:39.136Z 24 Acked 1023 Read 5 Sent Done Done false
111742024-02-08T06:43:39.136Z 25 Acked 1024 Write 5 Sent Done Done false
111752024-02-08T06:43:39.136Z 26 Acked 1025 Read 8 Sent Done Done false
111762024-02-08T06:43:39.136Z 27 Acked 1026 Write 5 Sent Done Done false
111772024-02-08T06:43:39.136Z 28 Acked 1027 Read 3 Sent Done Done false
111782024-02-08T06:43:39.136Z 29 Acked 1028 Read 2 Sent Done Done false
111792024-02-08T06:43:39.136Z 30 Acked 1029 Write 6 Sent Sent Sent false
111802024-02-08T06:43:39.137Z STATES DS:0 DS:1 DS:2 TOTAL
111812024-02-08T06:43:39.137Z New 0 0 0 0
111822024-02-08T06:43:39.137Z Sent 30 1 1 32
111832024-02-08T06:43:39.137Z Done 0 29 29 58
111842024-02-08T06:43:39.137Z Skipped 0 0 0 0
111852024-02-08T06:43:39.137Z Error 0 0 0 0
111862024-02-08T06:43:39.137ZLast Flush: 0 1017 1017
111872024-02-08T06:43:39.137ZDownstairs last five completed:
111882024-02-08T06:43:39.137ZUpstairs last five completed: 30 29 28 27 26
111892024-02-08T06:43:39.137Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:38.363318196Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1546}
111902024-02-08T06:43:39.137ZWrote out file "/tmp/test_repair_verify.data"
111912024-02-08T06:43:39.141Z
111922024-02-08T06:43:39.141ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111932024-02-08T06:43:39.156ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111942024-02-08T06:43:39.156Z 0 000-019 95 95 95 153 153 153 F F F
111952024-02-08T06:43:39.156Z 1 020-039 100 100 100 161 161 161 F F F
111962024-02-08T06:43:39.156Z 2 040-059 100 100 100 161 161 161 F T T <---
111972024-02-08T06:43:39.157Z 3 060-079 93 93 93 148 148 148 F F F
111982024-02-08T06:43:39.157Z 4 080-099 98 98 98 157 157 157 F F F
111992024-02-08T06:43:39.157Z 5 100-119 100 100 100 161 161 161 F F F
112002024-02-08T06:43:39.157Z 6 120-139 100 100 100 161 161 161 F F F
112012024-02-08T06:43:39.160Z 7 140-159 99 99 99 160 160 160 F F F
112022024-02-08T06:43:39.160Z 8 160-179 99 99 99 160 160 160 F F F
112032024-02-08T06:43:39.160Z 9 180-199 99 99 99 160 160 160 F F F
112042024-02-08T06:43:39.160Z 10 200-219 100 100 100 161 161 161 F F F
112052024-02-08T06:43:39.160Z 11 220-239 94 94 94 152 152 152 F F F
112062024-02-08T06:43:39.160Z 12 240-259 98 102 102 158 166 166 F F F <---
112072024-02-08T06:43:39.160Z 13 260-279 101 102 102 162 166 166 F T T <---
112082024-02-08T06:43:39.160Z 14 280-299 101 101 101 162 162 162 F T T <---
112092024-02-08T06:43:39.160Z 15 300-319 97 97 97 156 156 156 F F F
112102024-02-08T06:43:39.160Z 16 320-339 98 98 98 159 159 159 F F F
112112024-02-08T06:43:39.161Z 17 340-359 98 102 102 159 166 166 F T T <---
112122024-02-08T06:43:39.161Z 18 360-379 94 94 94 150 150 150 F T T <---
112132024-02-08T06:43:39.161Z 19 380-399 99 99 99 160 160 160 F T T <---
112142024-02-08T06:43:39.161Z 20 400-419 100 100 100 161 161 161 F F F
112152024-02-08T06:43:39.161Z 21 420-439 96 96 96 155 155 155 F F F
112162024-02-08T06:43:39.161Z 22 440-459 101 102 102 162 166 166 F F F <---
112172024-02-08T06:43:39.161Z 23 460-479 101 101 101 162 162 162 F F F
112182024-02-08T06:43:39.161Z 24 480-499 100 100 100 161 161 161 F F F
112192024-02-08T06:43:39.161Z 25 500-519 97 102 102 156 163 163 F F F <---
112202024-02-08T06:43:39.161Z 26 520-539 100 100 100 161 161 161 F F F
112212024-02-08T06:43:39.161Z 27 540-559 92 92 92 147 147 147 F T T <---
112222024-02-08T06:43:39.161Z 28 560-579 99 102 102 160 166 166 F F F <---
112232024-02-08T06:43:39.161Z 29 580-599 99 99 99 160 160 160 F F F
112242024-02-08T06:43:39.161ZMax gen: 102, Max flush: 166
112252024-02-08T06:43:39.161ZError: Difference in extent metadata found!
112262024-02-08T06:43:39.161ZA Difference in extent metadata is expected here
112272024-02-08T06:43:39.162ZOn loop 51
112282024-02-08T06:43:39.162Z
112292024-02-08T06:43:39.165ZVerifying data now
112302024-02-08T06:43:43.745ZNo change after verify
112312024-02-08T06:43:43.745ZLoop: 51 Downstairs dump after verify (and repair):
112322024-02-08T06:43:43.760ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112332024-02-08T06:43:43.760Z 0 000-019 95 95 95 153 153 153 F F F
112342024-02-08T06:43:43.760Z 1 020-039 100 100 100 161 161 161 F F F
112352024-02-08T06:43:43.760Z 2 040-059 103 103 103 167 167 167 F F F
112362024-02-08T06:43:43.764Z 3 060-079 93 93 93 148 148 148 F F F
112372024-02-08T06:43:43.764Z 4 080-099 98 98 98 157 157 157 F F F
112382024-02-08T06:43:43.764Z 5 100-119 100 100 100 161 161 161 F F F
112392024-02-08T06:43:43.764Z 6 120-139 100 100 100 161 161 161 F F F
112402024-02-08T06:43:43.764Z 7 140-159 99 99 99 160 160 160 F F F
112412024-02-08T06:43:43.764Z 8 160-179 99 99 99 160 160 160 F F F
112422024-02-08T06:43:43.764Z 9 180-199 99 99 99 160 160 160 F F F
112432024-02-08T06:43:43.764Z 10 200-219 100 100 100 161 161 161 F F F
112442024-02-08T06:43:43.764Z 11 220-239 94 94 94 152 152 152 F F F
112452024-02-08T06:43:43.765Z 12 240-259 102 102 102 166 166 166 F F F
112462024-02-08T06:43:43.765Z 13 260-279 103 103 103 167 167 167 F F F
112472024-02-08T06:43:43.765Z 14 280-299 103 103 103 167 167 167 F F F
112482024-02-08T06:43:43.765Z 15 300-319 97 97 97 156 156 156 F F F
112492024-02-08T06:43:43.765Z 16 320-339 98 98 98 159 159 159 F F F
112502024-02-08T06:43:43.765Z 17 340-359 103 103 103 167 167 167 F F F
112512024-02-08T06:43:43.765Z 18 360-379 103 103 103 167 167 167 F F F
112522024-02-08T06:43:43.765Z 19 380-399 103 103 103 167 167 167 F F F
112532024-02-08T06:43:43.765Z 20 400-419 100 100 100 161 161 161 F F F
112542024-02-08T06:43:43.765Z 21 420-439 96 96 96 155 155 155 F F F
112552024-02-08T06:43:43.765Z 22 440-459 102 102 102 166 166 166 F F F
112562024-02-08T06:43:43.765Z 23 460-479 101 101 101 162 162 162 F F F
112572024-02-08T06:43:43.765Z 24 480-499 100 100 100 161 161 161 F F F
112582024-02-08T06:43:43.765Z 25 500-519 102 102 102 163 163 163 F F F
112592024-02-08T06:43:43.765Z 26 520-539 100 100 100 161 161 161 F F F
112602024-02-08T06:43:43.766Z 27 540-559 103 103 103 167 167 167 F F F
112612024-02-08T06:43:43.766Z 28 560-579 102 102 102 166 166 166 F F F
112622024-02-08T06:43:43.766Z 29 580-599 99 99 99 160 160 160 F F F
112632024-02-08T06:43:43.766ZMax gen: 103, Max flush: 167
112642024-02-08T06:43:43.766Z
112652024-02-08T06:43:43.766ZBegin loop 52
112662024-02-08T06:43:43.766ZDownstairs to restart: 1
112672024-02-08T06:43:43.781Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012073737Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554}
112682024-02-08T06:43:43.781Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012332887Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554}
112692024-02-08T06:43:43.782Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012350306Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554}
112702024-02-08T06:43:43.787Z{"msg":"Crucible 8bbf907a-89a7-4b56-bc8d-815c0cf15467 has session id: 13b5eb8c-30aa-4661-8239-e1d9f392e5e4","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012368195Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112712024-02-08T06:43:43.787Z{"msg":"Crucible runtime is spawned
112722024-02-08T06:43:43.787ZUpstairs opts: Upstairs UUID: 8bbf907a-89a7-4b56-bc8d-815c0cf15467, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012409542Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112732024-02-08T06:43:43.787Z{"msg":"Crucible stats registered with UUID: 8bbf907a-89a7-4b56-bc8d-815c0cf15467","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.01243365Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112742024-02-08T06:43:43.787Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012468717Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554}
112752024-02-08T06:43:43.787Z{"msg":"8bbf907a-89a7-4b56-bc8d-815c0cf15467 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012534723Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112762024-02-08T06:43:43.787Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.01256503Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112772024-02-08T06:43:43.787Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012607337Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112782024-02-08T06:43:43.787Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012634715Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112792024-02-08T06:43:43.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012661573Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112802024-02-08T06:43:43.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012685891Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112812024-02-08T06:43:43.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012714649Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112822024-02-08T06:43:43.788Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012750056Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112832024-02-08T06:43:43.788Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012777324Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112842024-02-08T06:43:43.788Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012804322Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"io task","client":"2","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112852024-02-08T06:43:43.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.0128377Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112862024-02-08T06:43:43.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.012864268Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"io task","client":"2","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112872024-02-08T06:43:43.788Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:43.013064353Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112882024-02-08T06:43:43.788Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:43.013117499Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112892024-02-08T06:43:43.788Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:43.01322538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112902024-02-08T06:43:43.788Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.013287376Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112912024-02-08T06:43:43.788Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.013319024Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112922024-02-08T06:43:43.788Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.013428005Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112932024-02-08T06:43:43.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.013664058Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112942024-02-08T06:43:43.788Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.013696945Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112952024-02-08T06:43:43.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.013716304Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112962024-02-08T06:43:43.789Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.013813156Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112972024-02-08T06:43:43.789Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.014214756Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112982024-02-08T06:43:43.789Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.014343177Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
112992024-02-08T06:43:43.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.014726618Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113002024-02-08T06:43:43.789Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.014777844Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113012024-02-08T06:43:43.789Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.014805662Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113022024-02-08T06:43:43.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.014847479Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113032024-02-08T06:43:43.789Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.014873307Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113042024-02-08T06:43:43.789Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:43.014896825Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113052024-02-08T06:43:44.801Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.014492118Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113062024-02-08T06:43:44.801Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.014638496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113072024-02-08T06:43:44.806Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.015844036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113082024-02-08T06:43:44.806Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.015880313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113092024-02-08T06:43:44.807Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.016417623Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113102024-02-08T06:43:44.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.01699032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113112024-02-08T06:43:44.807Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017031967Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113122024-02-08T06:43:44.807Z{"msg":"[0]R flush_numbers[0..12]: [153, 161, 167, 148, 157, 161, 161, 160, 160, 160, 161, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017058275Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113132024-02-08T06:43:44.807Z{"msg":"[0]R generation[0..12]: [95, 100, 103, 93, 98, 100, 100, 99, 99, 99, 100, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017096092Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113142024-02-08T06:43:44.807Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.01712579Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113152024-02-08T06:43:44.807Z{"msg":"[1]R flush_numbers[0..12]: [153, 161, 167, 148, 157, 161, 161, 160, 160, 160, 161, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017141879Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113162024-02-08T06:43:44.807Z{"msg":"[1]R generation[0..12]: [95, 100, 103, 93, 98, 100, 100, 99, 99, 99, 100, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017163137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113172024-02-08T06:43:44.807Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017178276Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113182024-02-08T06:43:44.807Z{"msg":"[2]R flush_numbers[0..12]: [153, 161, 167, 148, 157, 161, 161, 160, 160, 160, 161, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017205014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113192024-02-08T06:43:44.807Z{"msg":"[2]R generation[0..12]: [95, 100, 103, 93, 98, 100, 100, 99, 99, 99, 100, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017226012Z","hostname":Wait for a query_work_queue command to finish before sending IO
113202024-02-08T06:43:44.807Z"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113212024-02-08T06:43:44.807Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.01725566Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113222024-02-08T06:43:44.807Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017269269Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113232024-02-08T06:43:44.807Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017291207Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113242024-02-08T06:43:44.808Z{"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017304646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113252024-02-08T06:43:44.808Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017336224Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113262024-02-08T06:43:44.808Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017364162Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113272024-02-08T06:43:44.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.01739582Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113282024-02-08T06:43:44.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017432977Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113292024-02-08T06:43:44.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017463964Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113302024-02-08T06:43:44.808Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017484203Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113312024-02-08T06:43:44.808Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.0175154Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113322024-02-08T06:43:44.808Z{"msg":"8bbf907a-89a7-4b56-bc8d-815c0cf15467 is now active with session: 13b5eb8c-30aa-4661-8239-e1d9f392e5e4","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017549628Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113332024-02-08T06:43:44.808Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:44.017578216Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554,"session_id":"13b5eb8c-30aa-4661-8239-e1d9f392e5e4"}
113342024-02-08T06:43:44.808Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
113352024-02-08T06:43:44.808Zlevel":30,"time":"2024-02-08T06:43:44.017615043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554}
113362024-02-08T06:43:44.808ZLoading write count information from file "/tmp/test_repair_verify.data"
113372024-02-08T06:43:44.808ZRun Repair workload
113382024-02-08T06:43:44.808Z01/30 Write block 305 len 24576 data: 10 8 5 4 5 6
113392024-02-08T06:43:44.809Z02/30 Write block 389 len 40960 data: 12 8 8 9 6 6 5 3 4 5
113402024-02-08T06:43:44.809Z03/30 Write block 528 len 32768 data: 9 9 10 11 11 10 10 10
113412024-02-08T06:43:44.809Z04/30 Write block 102 len 4096 data: 10
113422024-02-08T06:43:44.809Z05/30 Read block 327 len 28672
113432024-02-08T06:43:44.809Z06/30 Write block 56 len 8192 data: 12 10
113442024-02-08T06:43:44.809Z07/30 Write block 197 len 32768 data: 11 12 12 11 10 11 9 9
113452024-02-08T06:43:44.809Z08/30 Write block 151 len 36864 data: 8 7 8 9 9 10 9 10 9
113462024-02-08T06:43:44.809Z09/30 Flush
113472024-02-08T06:43:44.809Z10/30 Write block 143 len 28672 data: 5 5 5 4 5 6 8
113482024-02-08T06:43:44.809Z11/30 Read block 458 len 8192
113492024-02-08T06:43:44.809Z12/30 Write block 181 len 4096 data: 9
113502024-02-08T06:43:44.809Z13/30 Write block 581 len 28672 data: 8 9 9 9 8 7 8
113512024-02-08T06:43:44.809Z14/30 Write block 199 len 8192 data: 13 12
113522024-02-08T06:43:44.809Z15/30 Write block 147 len 32768 data: 6 7 9 7 9 8 9 10
113532024-02-08T06:43:44.809Z16/30 Write block 369 len 20480 data: 9 9 9 12 11
113542024-02-08T06:43:44.810Z17/30 Read block 407 len 28672
113552024-02-08T06:43:44.810Z18/30 Write block 172 len 8192 data: 10 12
113562024-02-08T06:43:44.810Z19/30 Flush
113572024-02-08T06:43:44.813Z20/30 Write block 585 len 12288 data: 9 8 9
113582024-02-08T06:43:44.813Z21/30 Write block 278 len 20480 data: 7 10 8 8 7
113592024-02-08T06:43:44.814Z22/30 Write block 397 len 24576 data: 5 6 7 7 9 9
113602024-02-08T06:43:44.814Z23/30 Write block 282 len 20480 data: 8 8 6 6 5
113612024-02-08T06:43:44.814Z24/30 Read block 470 len 32768
113622024-02-08T06:43:44.814Z25/30 Read block 395 len 40960
113632024-02-08T06:43:44.814Z26/30 Write block 74 len 28672 data: 8 6 7 4 5 6 7
113642024-02-08T06:43:44.814Z27/30 Write block 571 len 20480 data: 8 8 8 7 7
113652024-02-08T06:43:44.814Z28/30 Read block 429 len 8192
113662024-02-08T06:43:44.814Z29/30 Read block 512 len 12288
113672024-02-08T06:43:44.818Z30/30 Write block 259 len 40960 data: 9 10 14 12 10 10 11 10 11 11
113682024-02-08T06:43:44.818Z----------------------------------------------------------------
113692024-02-08T06:43:44.818Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
113702024-02-08T06:43:44.818ZStates: Active Active Active
113712024-02-08T06:43:44.818ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
113722024-02-08T06:43:44.818Z 1 Acked 1000 Write 6 Done Sent Done false
113732024-02-08T06:43:44.818Z 2 Acked 1001 Write 10 Done Sent Done false
113742024-02-08T06:43:44.818Z 3 Acked 1002 Write 8 Done Sent Done false
113752024-02-08T06:43:44.818Z 4 Acked 1003 Write 1 Done Sent Done false
113762024-02-08T06:43:44.818Z 5 Acked 1004 Read 7 Done Sent Done false
113772024-02-08T06:43:44.818Z 6 Acked 1005 Write 2 Done Sent Done false
113782024-02-08T06:43:44.818Z 7 Acked 1006 Write 8 Done Sent Done false
113792024-02-08T06:43:44.818Z 8 Acked 1007 Write 9 Done Sent Done false
113802024-02-08T06:43:44.818Z 9 Acked 1008 Flush 0 Done Sent Done false
113812024-02-08T06:43:44.819Z 10 Acked 1009 Write 7 Done Sent Done false
113822024-02-08T06:43:44.819Z 11 Acked 1010 Read 2 Done Sent Done false
113832024-02-08T06:43:44.819Z 12 Acked 1011 Write 1 Done Sent Done false
113842024-02-08T06:43:44.819Z 13 Acked 1012 Write 7 Done Sent Done false
113852024-02-08T06:43:44.819Z 14 Acked 1013 Write 2 Done Sent Done false
113862024-02-08T06:43:44.819Z 15 Acked 1014 Write 8 Done Sent Done false
113872024-02-08T06:43:44.819Z 16 Acked 1015 Write 5 Done Sent Done false
113882024-02-08T06:43:44.819Z 17 Acked 1016 Read 7 Done Sent Done false
113892024-02-08T06:43:44.819Z 18 Acked 1017 Write 2 Done Sent Done false
113902024-02-08T06:43:44.819Z 19 Acked 1018 Flush 0 Done Sent Done false
113912024-02-08T06:43:44.819Z 20 Acked 1019 Write 3 Done Sent Done false
113922024-02-08T06:43:44.819Z 21 Acked 1020 Write 5 Done Sent Done false
113932024-02-08T06:43:44.819Z 22 Acked 1021 Write 6 Done Sent Done false
113942024-02-08T06:43:44.819Z 23 Acked 1022 Write 5 Done Sent Done false
113952024-02-08T06:43:44.819Z 24 Acked 1023 Read 8 Done Sent Done false
113962024-02-08T06:43:44.820Z 25 Acked 1024 Read 10 Done Sent Done false
113972024-02-08T06:43:44.820Z 26 Acked 1025 Write 7 Done Sent Done false
113982024-02-08T06:43:44.820Z 27 Acked 1026 Write 5 Done Sent Done false
113992024-02-08T06:43:44.820Z 28 Acked 1027 Read 2 Done Sent Done false
114002024-02-08T06:43:44.820Z 29 Acked 1028 Read 3 Done Sent Done false
114012024-02-08T06:43:44.820Z 30 Acked 1029 Write 10 Sent Sent Sent false
114022024-02-08T06:43:44.820Z STATES DS:0 DS:1 DS:2 TOTAL
114032024-02-08T06:43:44.820Z New 0 0 0 0
114042024-02-08T06:43:44.820Z Sent 1 30 1 32
114052024-02-08T06:43:44.820Z Done 29 0 29 58
114062024-02-08T06:43:44.820Z Skipped 0 0 0 0
114072024-02-08T06:43:44.820Z Error 0 0 0 0
114082024-02-08T06:43:44.820ZLast Flush: 1018 0 1018
114092024-02-08T06:43:44.820ZDownstairs last five completed:
114102024-02-08T06:43:44.820ZUpstairs last five completed: 30 29 28 27 26
114112024-02-08T06:43:44.820Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:44.045906549Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1554}
114122024-02-08T06:43:44.821ZWrote out file "/tmp/test_repair_verify.data"
114132024-02-08T06:43:44.824Z
114142024-02-08T06:43:44.824ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
114152024-02-08T06:43:44.841ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114162024-02-08T06:43:44.841Z 0 000-019 95 95 95 153 153 153 F F F
114172024-02-08T06:43:44.841Z 1 020-039 100 100 100 161 161 161 F F F
114182024-02-08T06:43:44.842Z 2 040-059 104 103 104 168 167 168 F F F <---
114192024-02-08T06:43:44.842Z 3 060-079 93 93 93 148 148 148 T F T <---
114202024-02-08T06:43:44.842Z 4 080-099 98 98 98 157 157 157 T F T <---
114212024-02-08T06:43:44.842Z 5 100-119 104 100 104 168 161 168 F F F <---
114222024-02-08T06:43:44.842Z 6 120-139 100 100 100 161 161 161 F F F
114232024-02-08T06:43:44.842Z 7 140-159 104 99 104 169 160 169 F F F <---
114242024-02-08T06:43:44.842Z 8 160-179 104 99 104 169 160 169 F F F <---
114252024-02-08T06:43:44.842Z 9 180-199 104 99 104 169 160 169 F F F <---
114262024-02-08T06:43:44.845Z 10 200-219 104 100 104 169 161 169 F F F <---
114272024-02-08T06:43:44.845Z 11 220-239 94 94 94 152 152 152 F F F
114282024-02-08T06:43:44.845Z 12 240-259 102 102 102 166 166 166 T F T <---
114292024-02-08T06:43:44.846Z 13 260-279 103 103 103 167 167 167 T F T <---
114302024-02-08T06:43:44.846Z 14 280-299 103 103 103 167 167 167 T F T <---
114312024-02-08T06:43:44.846Z 15 300-319 104 97 104 168 156 168 F F F <---
114322024-02-08T06:43:44.846Z 16 320-339 98 98 98 159 159 159 F F F
114332024-02-08T06:43:44.846Z 17 340-359 103 103 103 167 167 167 F F F
114342024-02-08T06:43:44.846Z 18 360-379 104 103 104 169 167 169 F F F <---
114352024-02-08T06:43:44.846Z 19 380-399 104 103 104 168 167 168 T F T <---
114362024-02-08T06:43:44.846Z 20 400-419 100 100 100 161 161 161 T F T <---
114372024-02-08T06:43:44.846Z 21 420-439 96 96 96 155 155 155 F F F
114382024-02-08T06:43:44.846Z 22 440-459 102 102 102 166 166 166 F F F
114392024-02-08T06:43:44.846Z 23 460-479 101 101 101 162 162 162 F F F
114402024-02-08T06:43:44.846Z 24 480-499 100 100 100 161 161 161 F F F
114412024-02-08T06:43:44.846Z 25 500-519 102 102 102 163 163 163 F F F
114422024-02-08T06:43:44.846Z 26 520-539 104 100 104 168 161 168 F F F <---
114432024-02-08T06:43:44.846Z 27 540-559 103 103 103 167 167 167 F F F
114442024-02-08T06:43:44.847Z 28 560-579 102 102 102 166 166 166 T F T <---
114452024-02-08T06:43:44.847Z 29 580-599 104 99 104 169 160 169 T F T <---
114462024-02-08T06:43:44.847ZMax gen: 104, Max flush: 169
114472024-02-08T06:43:44.847ZError: Difference in extent metadata found!
114482024-02-08T06:43:44.847ZA Difference in extent metadata is expected here
114492024-02-08T06:43:44.847ZOn loop 52
114502024-02-08T06:43:44.847Z
114512024-02-08T06:43:44.850ZVerifying data now
114522024-02-08T06:43:49.689ZNo change after verify
114532024-02-08T06:43:49.689ZLoop: 52 Downstairs dump after verify (and repair):
114542024-02-08T06:43:49.703ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114552024-02-08T06:43:49.703Z 0 000-019 95 95 95 153 153 153 F F F
114562024-02-08T06:43:49.703Z 1 020-039 100 100 100 161 161 161 F F F
114572024-02-08T06:43:49.704Z 2 040-059 104 104 104 168 168 168 F F F
114582024-02-08T06:43:49.707Z 3 060-079 105 105 105 170 170 170 F F F
114592024-02-08T06:43:49.707Z 4 080-099 105 105 105 170 170 170 F F F
114602024-02-08T06:43:49.707Z 5 100-119 104 104 104 168 168 168 F F F
114612024-02-08T06:43:49.707Z 6 120-139 100 100 100 161 161 161 F F F
114622024-02-08T06:43:49.707Z 7 140-159 104 104 104 169 169 169 F F F
114632024-02-08T06:43:49.707Z 8 160-179 104 104 104 169 169 169 F F F
114642024-02-08T06:43:49.707Z 9 180-199 104 104 104 169 169 169 F F F
114652024-02-08T06:43:49.707Z 10 200-219 104 104 104 169 169 169 F F F
114662024-02-08T06:43:49.707Z 11 220-239 94 94 94 152 152 152 F F F
114672024-02-08T06:43:49.708Z 12 240-259 105 105 105 170 170 170 F F F
114682024-02-08T06:43:49.708Z 13 260-279 105 105 105 170 170 170 F F F
114692024-02-08T06:43:49.708Z 14 280-299 105 105 105 170 170 170 F F F
114702024-02-08T06:43:49.708Z 15 300-319 104 104 104 168 168 168 F F F
114712024-02-08T06:43:49.708Z 16 320-339 98 98 98 159 159 159 F F F
114722024-02-08T06:43:49.708Z 17 340-359 103 103 103 167 167 167 F F F
114732024-02-08T06:43:49.708Z 18 360-379 104 104 104 169 169 169 F F F
114742024-02-08T06:43:49.708Z 19 380-399 105 105 105 170 170 170 F F F
114752024-02-08T06:43:49.708Z 20 400-419 105 105 105 170 170 170 F F F
114762024-02-08T06:43:49.708Z 21 420-439 96 96 96 155 155 155 F F F
114772024-02-08T06:43:49.708Z 22 440-459 102 102 102 166 166 166 F F F
114782024-02-08T06:43:49.708Z 23 460-479 101 101 101 162 162 162 F F F
114792024-02-08T06:43:49.708Z 24 480-499 100 100 100 161 161 161 F F F
114802024-02-08T06:43:49.708Z 25 500-519 102 102 102 163 163 163 F F F
114812024-02-08T06:43:49.708Z 26 520-539 104 104 104 168 168 168 F F F
114822024-02-08T06:43:49.709Z 27 540-559 103 103 103 167 167 167 F F F
114832024-02-08T06:43:49.709Z 28 560-579 105 105 105 170 170 170 F F F
114842024-02-08T06:43:49.709Z 29 580-599 105 105 105 170 170 170 F F F
114852024-02-08T06:43:49.709ZMax gen: 105, Max flush: 170
114862024-02-08T06:43:49.709Z
114872024-02-08T06:43:49.709ZBegin loop 53
114882024-02-08T06:43:49.709ZDownstairs to restart: 1
114892024-02-08T06:43:49.724Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.954367003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562}
114902024-02-08T06:43:49.729Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.95466357Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562}
114912024-02-08T06:43:49.729Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.954697428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562}
114922024-02-08T06:43:49.729Z{"msg":"Crucible b6b50dbe-5765-40a3-89c4-25fa0a48fef5 has session id: b2282ed0-f27b-4e43-98bf-394cfaa6a05f","v":0,"name":"crucible","level":30Crucible runtime is spawned
114932024-02-08T06:43:49.729Z,"time":"2024-02-08T06:43:48.954733755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
114942024-02-08T06:43:49.730Z{"msg":"Upstairs opts: Upstairs UUID: b6b50dbe-5765-40a3-89c4-25fa0a48fef5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.954771812Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
114952024-02-08T06:43:49.730Z{"msg":"Crucible stats registered with UUID: b6b50dbe-5765-40a3-89c4-25fa0a48fef5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.954815579Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
114962024-02-08T06:43:49.730Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.954839407Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562}
114972024-02-08T06:43:49.730Z{"msg":"b6b50dbe-5765-40a3-89c4-25fa0a48fef5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.954947069Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
114982024-02-08T06:43:49.730Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.954996605Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
114992024-02-08T06:43:49.730Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955021633Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115002024-02-08T06:43:49.730Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955047501Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115012024-02-08T06:43:49.730Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.95507205Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115022024-02-08T06:43:49.730Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955094788Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115032024-02-08T06:43:49.730Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955119086Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115042024-02-08T06:43:49.730Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955146034Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115052024-02-08T06:43:49.730Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955185461Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115062024-02-08T06:43:49.730Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955220628Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115072024-02-08T06:43:49.730Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955254976Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115082024-02-08T06:43:49.730Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955369687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115092024-02-08T06:43:49.731Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955429353Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115102024-02-08T06:43:49.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.955942504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115112024-02-08T06:43:49.731Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.956024868Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115122024-02-08T06:43:49.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.956075074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115132024-02-08T06:43:49.731Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.956093943Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115142024-02-08T06:43:49.731Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.956613194Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115152024-02-08T06:43:49.731Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.9566598Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115162024-02-08T06:43:49.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.956994735Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115172024-02-08T06:43:49.731Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.957025693Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115182024-02-08T06:43:49.731Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.957242177Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115192024-02-08T06:43:49.731Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.957289123Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115202024-02-08T06:43:49.731Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.957308322Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115212024-02-08T06:43:49.731Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.95733523Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115222024-02-08T06:43:49.731Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.957358428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115232024-02-08T06:43:49.731Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.957373207Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115242024-02-08T06:43:49.732Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.957591041Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115252024-02-08T06:43:49.736Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958037187Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115262024-02-08T06:43:49.737Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958078774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115272024-02-08T06:43:49.737Z{"msg":"[0]R flush_numbers[0..12]: [153, 161, 168, 170, 170, 168, 161, 169, 169, 169, 169, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958095532Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115282024-02-08T06:43:49.737Z{"msg":"[0]R generation[0..12]: [95, 100, 104, 105, 105, 104, 100, 104, 104, 104, 104, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.95812887Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115292024-02-08T06:43:49.737Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958159498Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115302024-02-08T06:43:49.737Z{"msg":"[1]R flush_numbers[0..12]: [153, 161, 168, 170, 170, 168, 161, 169, 169, 169, 169, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958175886Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115312024-02-08T06:43:49.737Z{"msg":"[1]R generation[0..12]: [95, 100, 104, 105, 105, 104, 100, 104, 104, 104, 104, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958190265Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115322024-02-08T06:43:49.737Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
115332024-02-08T06:43:49.737Z,"time":"2024-02-08T06:43:48.958204624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115342024-02-08T06:43:49.737Z{"msg":"[2]R flush_numbers[0..12]: [153, 161, 168, 170, 170, 168, 161, 169, 169, 169, 169, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958229993Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115352024-02-08T06:43:49.737Z{"msg":"[2]R generation[0..12]: [95, 100, 104, 105, 105, 104, 100, 104, 104, 104, 104, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958243881Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115362024-02-08T06:43:49.737Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.95826584Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115372024-02-08T06:43:49.737Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958279829Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115382024-02-08T06:43:49.737Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958301587Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115392024-02-08T06:43:49.737Z{"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958315646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115402024-02-08T06:43:49.737Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958329695Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115412024-02-08T06:43:49.738Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958350763Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115422024-02-08T06:43:49.738Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958379071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115432024-02-08T06:43:49.738Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.95839462Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115442024-02-08T06:43:49.738Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958416948Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115452024-02-08T06:43:49.738Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958446216Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115462024-02-08T06:43:49.738Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958461485Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115472024-02-08T06:43:49.738Z{"msg":"b6b50dbe-5765-40a3-89c4-25fa0a48fef5 is now active with session: b2282ed0-f27b-4e43-98bf-394cfaa6a05f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958491283Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115482024-02-08T06:43:49.738Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958505762Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562,"session_id":"b2282ed0-f27b-4e43-98bf-394cfaa6a05f"}
115492024-02-08T06:43:49.738Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:48.958535579Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562}
115502024-02-08T06:43:49.738ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
115512024-02-08T06:43:49.738ZLoading write count information from file "/tmp/test_repair_verify.data"
115522024-02-08T06:43:49.738ZRun Repair workload
115532024-02-08T06:43:49.738Z01/30 Write block 419 len 20480 data: 8 8 7 8 7
115542024-02-08T06:43:49.738Z02/30 Read block 352 len 16384
115552024-02-08T06:43:49.738Z03/30 Write block 183 len 28672 data: 6 6 9 9 8 8 7
115562024-02-08T06:43:49.738Z04/30 Write block 319 len 32768 data: 9 13 11 12 12 10 11 9
115572024-02-08T06:43:49.739Z05/30 Write block 382 len 12288 data: 7 7 10
115582024-02-08T06:43:49.739Z06/30 Write block 171 len 8192 data: 9 11
115592024-02-08T06:43:49.739Z07/30 Read block 596 len 16384
115602024-02-08T06:43:49.739Z08/30 Flush
115612024-02-08T06:43:49.742Z09/30 Write block 300 len 40960 data: 10 12 12 11 10 11 9 6 5 6
115622024-02-08T06:43:49.742Z10/30 Write block 300 len 12288 data: 11 13 13
115632024-02-08T06:43:49.742Z11/30 Read block 388 len 24576
115642024-02-08T06:43:49.742Z12/30 Write block 444 len 24576 data: 7 7 6 7 8 9
115652024-02-08T06:43:49.742Z13/30 Read block 146 len 12288
115662024-02-08T06:43:49.743Z14/30 Read block 27 len 4096
115672024-02-08T06:43:49.746Z15/30 Read block 471 len 24576
115682024-02-08T06:43:49.746Z16/30 Read block 14 len 12288
115692024-02-08T06:43:49.746Z17/30 Read block 351 len 32768
115702024-02-08T06:43:49.746Z18/30 Write block 167 len 12288 data: 15 12 13
115712024-02-08T06:43:49.746Z19/30 Read block 227 len 12288
115722024-02-08T06:43:49.749Z20/30 Read block 532 len 4096
115732024-02-08T06:43:49.749Z21/30 Read block 190 len 40960
115742024-02-08T06:43:49.749Z22/30 Read block 185 len 24576
115752024-02-08T06:43:49.749Z23/30 Read block 493 len 40960
115762024-02-08T06:43:49.752Z24/30 Write block 437 len 36864 data: 11 9 6 8 8 8 9 8 8
115772024-02-08T06:43:49.752Z25/30 Write block 246 len 40960 data: 8 8 9 12 12 12 9 7 8 10
115782024-02-08T06:43:49.752Z26/30 Read block 509 len 36864
115792024-02-08T06:43:49.755Z27/30 Read block 537 len 36864
115802024-02-08T06:43:49.755Z28/30 Write block 493 len 8192 data: 10 8
115812024-02-08T06:43:49.756Z29/30 Write block 566 len 32768 data: 8 8 8 7 9 9 9 9
115822024-02-08T06:43:49.756Z30/30 Write block 498 len 4096 data: 7
115832024-02-08T06:43:49.759Z----------------------------------------------------------------
115842024-02-08T06:43:49.759Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
115852024-02-08T06:43:49.759ZStates: Active Active Active
115862024-02-08T06:43:49.759ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
115872024-02-08T06:43:49.759Z 1 Acked 1000 Write 5 Done Done Done false
115882024-02-08T06:43:49.759Z 2 Acked 1001 Read 4 Done Done Done false
115892024-02-08T06:43:49.759Z 3 Acked 1002 Write 7 Done Sent Done false
115902024-02-08T06:43:49.759Z 4 Acked 1003 Write 8 Done Sent Done false
115912024-02-08T06:43:49.760Z 5 Acked 1004 Write 3 Done Sent Done false
115922024-02-08T06:43:49.760Z 6 Acked 1005 Write 2 Done Sent Done false
115932024-02-08T06:43:49.760Z 7 Acked 1006 Read 4 Done Sent Done false
115942024-02-08T06:43:49.760Z 8 Acked 1007 Flush 0 Done Sent Done false
115952024-02-08T06:43:49.760Z 9 Acked 1008 Write 10 Done Sent Done false
115962024-02-08T06:43:49.760Z 10 Acked 1009 Write 3 Done Sent Done false
115972024-02-08T06:43:49.760Z 11 Acked 1010 Read 6 Done Sent Done false
115982024-02-08T06:43:49.760Z 12 Acked 1011 Write 6 Done Sent Done false
115992024-02-08T06:43:49.760Z 13 Acked 1012 Read 3 Done Sent Done false
116002024-02-08T06:43:49.760Z 14 Acked 1013 Read 1 Done Sent Done false
116012024-02-08T06:43:49.760Z 15 Acked 1014 Read 6 Done Sent Done false
116022024-02-08T06:43:49.760Z 16 Acked 1015 Read 3 Done Sent Done false
116032024-02-08T06:43:49.760Z 17 Acked 1016 Read 8 Done Sent Done false
116042024-02-08T06:43:49.760Z 18 Acked 1017 Write 3 Done Sent Done false
116052024-02-08T06:43:49.760Z 19 Acked 1018 Read 3 Done Sent Done false
116062024-02-08T06:43:49.761Z 20 Acked 1019 Read 1 Done Sent Done false
116072024-02-08T06:43:49.761Z 21 Acked 1020 Read 10 Done Sent Done false
116082024-02-08T06:43:49.761Z 22 Acked 1021 Read 6 Done Sent Done false
116092024-02-08T06:43:49.761Z 23 Acked 1022 Read 10 Done Sent Done false
116102024-02-08T06:43:49.761Z 24 Acked 1023 Write 9 Done Sent Done false
116112024-02-08T06:43:49.761Z 25 Acked 1024 Write 10 Done Sent Done false
116122024-02-08T06:43:49.761Z 26 Acked 1025 Read 9 Done Sent Done false
116132024-02-08T06:43:49.761Z 27 Acked 1026 Read 9 Done Sent Done false
116142024-02-08T06:43:49.761Z 28 Acked 1027 Write 2 Done Sent Done false
116152024-02-08T06:43:49.761Z 29 Acked 1028 Write 8 Sent Sent Sent false
116162024-02-08T06:43:49.761Z 30 Acked 1029 Write 1 Sent Sent Sent false
116172024-02-08T06:43:49.761Z STATES DS:0 DS:1 DS:2 TOTAL
116182024-02-08T06:43:49.761Z New 0 0 0 0
116192024-02-08T06:43:49.761Z Sent 2 28 2 32
116202024-02-08T06:43:49.761Z Done 28 2 28 58
116212024-02-08T06:43:49.761Z Skipped 0 0 0 0
116222024-02-08T06:43:49.762Z Error 0 0 0 0
116232024-02-08T06:43:49.762ZLast Flush: 1007 0 1007
116242024-02-08T06:43:49.762ZDownstairs last five completed:
116252024-02-08T06:43:49.762ZUpstairs last five completed: 30 29 28 27 26
116262024-02-08T06:43:49.762Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:48.98649615Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1562}
116272024-02-08T06:43:49.762ZWrote out file "/tmp/test_repair_verify.data"
116282024-02-08T06:43:49.765Z
116292024-02-08T06:43:49.766ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
116302024-02-08T06:43:49.781ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116312024-02-08T06:43:49.781Z 0 000-019 95 95 95 153 153 153 F F F
116322024-02-08T06:43:49.782Z 1 020-039 100 100 100 161 161 161 F F F
116332024-02-08T06:43:49.782Z 2 040-059 104 104 104 168 168 168 F F F
116342024-02-08T06:43:49.782Z 3 060-079 105 105 105 170 170 170 F F F
116352024-02-08T06:43:49.782Z 4 080-099 105 105 105 170 170 170 F F F
116362024-02-08T06:43:49.782Z 5 100-119 104 104 104 168 168 168 F F F
116372024-02-08T06:43:49.782Z 6 120-139 100 100 100 161 161 161 F F F
116382024-02-08T06:43:49.782Z 7 140-159 104 104 104 169 169 169 F F F
116392024-02-08T06:43:49.782Z 8 160-179 106 104 106 171 169 171 T F T <---
116402024-02-08T06:43:49.782Z 9 180-199 106 104 106 171 169 171 F F F <---
116412024-02-08T06:43:49.782Z 10 200-219 104 104 104 169 169 169 F F F
116422024-02-08T06:43:49.782Z 11 220-239 94 94 94 152 152 152 F F F
116432024-02-08T06:43:49.782Z 12 240-259 105 105 105 170 170 170 T F T <---
116442024-02-08T06:43:49.782Z 13 260-279 105 105 105 170 170 170 F F F
116452024-02-08T06:43:49.783Z 14 280-299 105 105 105 170 170 170 F F F
116462024-02-08T06:43:49.789Z 15 300-319 106 104 106 171 168 171 T F T <---
116472024-02-08T06:43:49.789Z 16 320-339 106 98 106 171 159 171 F F F <---
116482024-02-08T06:43:49.789Z 17 340-359 103 103 103 167 167 167 F F F
116492024-02-08T06:43:49.789Z 18 360-379 104 104 104 169 169 169 F F F
116502024-02-08T06:43:49.789Z 19 380-399 106 105 106 171 170 171 F F F <---
116512024-02-08T06:43:49.789Z 20 400-419 106 105 106 171 170 171 F T F <---
116522024-02-08T06:43:49.789Z 21 420-439 106 96 106 171 155 171 T T T <---
116532024-02-08T06:43:49.789Z 22 440-459 102 102 102 166 166 166 T F T <---
116542024-02-08T06:43:49.789Z 23 460-479 101 101 101 162 162 162 F F F
116552024-02-08T06:43:49.789Z 24 480-499 100 100 100 161 161 161 T F T <---
116562024-02-08T06:43:49.789Z 25 500-519 102 102 102 163 163 163 F F F
116572024-02-08T06:43:49.789Z 26 520-539 104 104 104 168 168 168 F F F
116582024-02-08T06:43:49.790Z 27 540-559 103 103 103 167 167 167 F F F
116592024-02-08T06:43:49.790Z 28 560-579 105 105 105 170 170 170 T F T <---
116602024-02-08T06:43:49.790Z 29 580-599 105 105 105 170 170 170 F F F
116612024-02-08T06:43:49.790ZMax gen: 106, Max flush: 171
116622024-02-08T06:43:49.790ZError: Difference in extent metadata found!
116632024-02-08T06:43:49.790ZA Difference in extent metadata is expected here
116642024-02-08T06:43:49.790ZOn loop 53
116652024-02-08T06:43:49.790Z
116662024-02-08T06:43:49.793ZVerifying data now
116672024-02-08T06:43:55.345ZNo change after verify
116682024-02-08T06:43:55.345ZLoop: 53 Downstairs dump after verify (and repair):
116692024-02-08T06:43:55.360ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116702024-02-08T06:43:55.360Z 0 000-019 95 95 95 153 153 153 F F F
116712024-02-08T06:43:55.360Z 1 020-039 100 100 100 161 161 161 F F F
116722024-02-08T06:43:55.360Z 2 040-059 104 104 104 168 168 168 F F F
116732024-02-08T06:43:55.360Z 3 060-079 105 105 105 170 170 170 F F F
116742024-02-08T06:43:55.360Z 4 080-099 105 105 105 170 170 170 F F F
116752024-02-08T06:43:55.360Z 5 100-119 104 104 104 168 168 168 F F F
116762024-02-08T06:43:55.360Z 6 120-139 100 100 100 161 161 161 F F F
116772024-02-08T06:43:55.360Z 7 140-159 104 104 104 169 169 169 F F F
116782024-02-08T06:43:55.364Z 8 160-179 107 107 107 172 172 172 F F F
116792024-02-08T06:43:55.364Z 9 180-199 106 106 106 171 171 171 F F F
116802024-02-08T06:43:55.364Z 10 200-219 104 104 104 169 169 169 F F F
116812024-02-08T06:43:55.364Z 11 220-239 94 94 94 152 152 152 F F F
116822024-02-08T06:43:55.364Z 12 240-259 107 107 107 172 172 172 F F F
116832024-02-08T06:43:55.364Z 13 260-279 105 105 105 170 170 170 F F F
116842024-02-08T06:43:55.364Z 14 280-299 105 105 105 170 170 170 F F F
116852024-02-08T06:43:55.364Z 15 300-319 107 107 107 172 172 172 F F F
116862024-02-08T06:43:55.364Z 16 320-339 106 106 106 171 171 171 F F F
116872024-02-08T06:43:55.364Z 17 340-359 103 103 103 167 167 167 F F F
116882024-02-08T06:43:55.364Z 18 360-379 104 104 104 169 169 169 F F F
116892024-02-08T06:43:55.364Z 19 380-399 106 106 106 171 171 171 F F F
116902024-02-08T06:43:55.364Z 20 400-419 106 106 106 171 171 171 F F F
116912024-02-08T06:43:55.364Z 21 420-439 107 107 107 172 172 172 F F F
116922024-02-08T06:43:55.364Z 22 440-459 107 107 107 172 172 172 F F F
116932024-02-08T06:43:55.365Z 23 460-479 101 101 101 162 162 162 F F F
116942024-02-08T06:43:55.365Z 24 480-499 107 107 107 172 172 172 F F F
116952024-02-08T06:43:55.365Z 25 500-519 102 102 102 163 163 163 F F F
116962024-02-08T06:43:55.365Z 26 520-539 104 104 104 168 168 168 F F F
116972024-02-08T06:43:55.365Z 27 540-559 103 103 103 167 167 167 F F F
116982024-02-08T06:43:55.365Z 28 560-579 107 107 107 172 172 172 F F F
116992024-02-08T06:43:55.365Z 29 580-599 105 105 105 170 170 170 F F F
117002024-02-08T06:43:55.365ZMax gen: 107, Max flush: 172
117012024-02-08T06:43:55.365Z
117022024-02-08T06:43:55.365ZBegin loop 54
117032024-02-08T06:43:55.365ZDownstairs to restart: 0
117042024-02-08T06:43:55.380Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.60986352Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570}
117052024-02-08T06:43:55.380Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610138209Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570}
117062024-02-08T06:43:55.385Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610177226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570}
117072024-02-08T06:43:55.385Z{"msg":"Crucible edc01c41-9030-45e3-817f-e67fd52d8ec2 has session id: b9945524-1b91-416b-b393-7080b9684176","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610202834Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117082024-02-08T06:43:55.385Z{"msg":"Crucible runtime is spawned
117092024-02-08T06:43:55.386ZUpstairs opts: Upstairs UUID: edc01c41-9030-45e3-817f-e67fd52d8ec2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610250291Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117102024-02-08T06:43:55.386Z{"msg":"Crucible stats registered with UUID: edc01c41-9030-45e3-817f-e67fd52d8ec2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610278589Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117112024-02-08T06:43:55.386Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610307547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570}
117122024-02-08T06:43:55.386Z{"msg":"edc01c41-9030-45e3-817f-e67fd52d8ec2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610472034Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117132024-02-08T06:43:55.386Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610515631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117142024-02-08T06:43:55.386Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610541169Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117152024-02-08T06:43:55.386Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610583016Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117162024-02-08T06:43:55.386Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610610584Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117172024-02-08T06:43:55.386Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610636152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117182024-02-08T06:43:55.386Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.6106605Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117192024-02-08T06:43:55.386Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610685208Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"io task","client":"0","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117202024-02-08T06:43:55.386Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610720845Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"io task","client":"2","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117212024-02-08T06:43:55.386Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610746894Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117222024-02-08T06:43:55.386Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610782851Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"io task","client":"2","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117232024-02-08T06:43:55.386Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610810499Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117242024-02-08T06:43:55.387Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.610847306Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"io task","client":"0","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117252024-02-08T06:43:55.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.611487348Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117262024-02-08T06:43:55.387Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.611543964Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117272024-02-08T06:43:55.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.611573032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117282024-02-08T06:43:55.387Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.61159236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117292024-02-08T06:43:55.387Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.612152748Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117302024-02-08T06:43:55.387Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.612189385Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117312024-02-08T06:43:55.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.612746044Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117322024-02-08T06:43:55.387Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.61278772Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117332024-02-08T06:43:55.387Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.612805919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117342024-02-08T06:43:55.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.612855985Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117352024-02-08T06:43:55.387Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.612877414Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117362024-02-08T06:43:55.387Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.612908071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117372024-02-08T06:43:55.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.613139084Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117382024-02-08T06:43:55.387Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.613178471Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117392024-02-08T06:43:55.393Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.613638266Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117402024-02-08T06:43:55.393Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614078093Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117412024-02-08T06:43:55.393Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614136489Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117422024-02-08T06:43:55.393Z{"msg":"[0]R flush_numbers[0..12]: [153, 161, 168, 170, 170, 168, 161, 169, 172, 171, 169, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614174886Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117432024-02-08T06:43:55.393Z{"msg":"[0]R generation[0..12]: [95, 100, 104, 105, 105, 104, 100, 104, 107, 106, 104, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614211763Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117442024-02-08T06:43:55.393Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614246651Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117452024-02-08T06:43:55.393Z{"msg":"[1]R flush_numbers[0..12]: [153, 161, 168, 170, 170, 168, 161, 169, 172, 171, 169, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614282768Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117462024-02-08T06:43:55.393Z{"msg":"[1]R generation[0..12]: [95, 100, 104, 105, 105, 104, 100, 104, 107, 106, 104, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614321925Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117472024-02-08T06:43:55.393ZWait for a query_work_queue command to finish before sending IO
117482024-02-08T06:43:55.393Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614364252Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117492024-02-08T06:43:55.394Z{"msg":"[2]R flush_numbers[0..12]: [153, 161, 168, 170, 170, 168, 161, 169, 172, 171, 169, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.61438929Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117502024-02-08T06:43:55.394Z{"msg":"[2]R generation[0..12]: [95, 100, 104, 105, 105, 104, 100, 104, 107, 106, 104, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614423887Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117512024-02-08T06:43:55.394Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614468524Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117522024-02-08T06:43:55.394Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614505571Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117532024-02-08T06:43:55.394Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614544698Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117542024-02-08T06:43:55.394Z{"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614579006Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117552024-02-08T06:43:55.394Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614604574Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117562024-02-08T06:43:55.394Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614639591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117572024-02-08T06:43:55.394Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614665039Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117582024-02-08T06:43:55.394Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614701247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117592024-02-08T06:43:55.394Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614739314Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117602024-02-08T06:43:55.394Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614777231Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117612024-02-08T06:43:55.394Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614802349Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal",Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
117622024-02-08T06:43:55.394Z"pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117632024-02-08T06:43:55.395Z{"msg":"edc01c41-9030-45e3-817f-e67fd52d8ec2 is now active with session: b9945524-1b91-416b-b393-7080b9684176","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614841026Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117642024-02-08T06:43:55.395Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614871504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570,"session_id":"b9945524-1b91-416b-b393-7080b9684176"}
117652024-02-08T06:43:55.395Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:54.614889652Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570}
117662024-02-08T06:43:55.395ZLoading write count information from file "/tmp/test_repair_verify.data"
117672024-02-08T06:43:55.395ZRun Repair workload
117682024-02-08T06:43:55.395Z01/30 Write block 215 len 16384 data: 8 8 9 8
117692024-02-08T06:43:55.395Z02/30 Write block 458 len 8192 data: 10 11
117702024-02-08T06:43:55.395Z03/30 Read block 536 len 40960
117712024-02-08T06:43:55.395Z04/30 Read block 186 len 20480
117722024-02-08T06:43:55.395Z05/30 Read block 191 len 16384
117732024-02-08T06:43:55.395Z06/30 Write block 187 len 20480 data: 9 9 8 7 8
117742024-02-08T06:43:55.395Z07/30 Read block 374 len 40960
117752024-02-08T06:43:55.399Z08/30 Read block 487 len 8192
117762024-02-08T06:43:55.399Z09/30 Read block 193 len 24576
117772024-02-08T06:43:55.399Z10/30 Write block 284 len 4096 data: 7
117782024-02-08T06:43:55.399Z11/30 Read block 483 len 16384
117792024-02-08T06:43:55.399Z12/30 Write block 592 len 4096 data: 9
117802024-02-08T06:43:55.399Z13/30 Write block 279 len 8192 data: 11 9
117812024-02-08T06:43:55.399Z14/30 Read block 262 len 28672
117822024-02-08T06:43:55.399Z15/30 Read block 57 len 12288
117832024-02-08T06:43:55.399Z16/30 Read block 62 len 12288
117842024-02-08T06:43:55.400Z17/30 Flush
117852024-02-08T06:43:55.403Z18/30 Write block 452 len 16384 data: 7 8 7 10
117862024-02-08T06:43:55.403Z19/30 Read block 189 len 12288
117872024-02-08T06:43:55.403Z20/30 Write block 561 len 4096 data: 7
117882024-02-08T06:43:55.406Z21/30 Write block 211 len 32768 data: 11 13 12 11 9 9 10 9
117892024-02-08T06:43:55.406Z22/30 Read block 227 len 28672
117902024-02-08T06:43:55.406Z23/30 Flush
117912024-02-08T06:43:55.409Z24/30 Write block 530 len 28672 data: 11 12 12 11 11 11 11
117922024-02-08T06:43:55.409Z25/30 Read block 6 len 8192
117932024-02-08T06:43:55.409Z26/30 Read block 244 len 16384
117942024-02-08T06:43:55.413Z27/30 Read block 231 len 16384
117952024-02-08T06:43:55.413Z28/30 Read block 224 len 40960
117962024-02-08T06:43:55.413Z29/30 Read block 550 len 20480
117972024-02-08T06:43:55.413Z30/30 Write block 302 len 28672 data: 14 12 11 12 10 7 6
117982024-02-08T06:43:55.416Z----------------------------------------------------------------
117992024-02-08T06:43:55.416Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
118002024-02-08T06:43:55.416ZStates: Active Active Active
118012024-02-08T06:43:55.417ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
118022024-02-08T06:43:55.417Z 1 Acked 1000 Write 4 Sent Done Done false
118032024-02-08T06:43:55.417Z 2 Acked 1001 Write 2 Sent Done Done false
118042024-02-08T06:43:55.417Z 3 Acked 1002 Read 10 Sent Done Done false
118052024-02-08T06:43:55.417Z 4 Acked 1003 Read 5 Sent Done Done false
118062024-02-08T06:43:55.417Z 5 Acked 1004 Read 4 Sent Done Done false
118072024-02-08T06:43:55.417Z 6 Acked 1005 Write 5 Sent Done Done false
118082024-02-08T06:43:55.417Z 7 Acked 1006 Read 10 Sent Done Done false
118092024-02-08T06:43:55.417Z 8 Acked 1007 Read 2 Sent Done Done false
118102024-02-08T06:43:55.417Z 9 Acked 1008 Read 6 Sent Done Done false
118112024-02-08T06:43:55.417Z 10 Acked 1009 Write 1 Sent Done Done false
118122024-02-08T06:43:55.417Z 11 Acked 1010 Read 4 Sent Done Done false
118132024-02-08T06:43:55.417Z 12 Acked 1011 Write 1 Sent Done Done false
118142024-02-08T06:43:55.417Z 13 Acked 1012 Write 2 Sent Done Done false
118152024-02-08T06:43:55.417Z 14 Acked 1013 Read 7 Sent Done Done false
118162024-02-08T06:43:55.417Z 15 Acked 1014 Read 3 Sent Done Done false
118172024-02-08T06:43:55.418Z 16 Acked 1015 Read 3 Sent Done Done false
118182024-02-08T06:43:55.418Z 17 Acked 1016 Flush 0 Sent Done Done false
118192024-02-08T06:43:55.418Z 18 Acked 1017 Write 4 Sent Done Done false
118202024-02-08T06:43:55.418Z 19 Acked 1018 Read 3 Sent Done Done false
118212024-02-08T06:43:55.418Z 20 Acked 1019 Write 1 Sent Done Done false
118222024-02-08T06:43:55.418Z 21 Acked 1020 Write 8 Sent Done Done false
118232024-02-08T06:43:55.418Z 22 Acked 1021 Read 7 Sent Done Done false
118242024-02-08T06:43:55.418Z 23 Acked 1022 Flush 0 Sent Done Done false
118252024-02-08T06:43:55.418Z 24 Acked 1023 Write 7 Sent Done Done false
118262024-02-08T06:43:55.418Z 25 Acked 1024 Read 2 Sent Done Done false
118272024-02-08T06:43:55.418Z 26 Acked 1025 Read 4 Sent Done Done false
118282024-02-08T06:43:55.418Z 27 Acked 1026 Read 4 Sent Done Done false
118292024-02-08T06:43:55.418Z 28 Acked 1027 Read 10 Sent Done Done false
118302024-02-08T06:43:55.418Z 29 Acked 1028 Read 5 Sent Done Done false
118312024-02-08T06:43:55.418Z 30 Acked 1029 Write 7 Sent Sent Sent false
118322024-02-08T06:43:55.419Z STATES DS:0 DS:1 DS:2 TOTAL
118332024-02-08T06:43:55.419Z New 0 0 0 0
118342024-02-08T06:43:55.419Z Sent 30 1 1 32
118352024-02-08T06:43:55.419Z Done 0 29 29 58
118362024-02-08T06:43:55.419Z Skipped 0 0 0 0
118372024-02-08T06:43:55.419Z Error 0 0 0 0
118382024-02-08T06:43:55.419ZLast Flush: 0 1022 1022
118392024-02-08T06:43:55.419ZDownstairs last five completed:
118402024-02-08T06:43:55.419ZUpstairs last five completed: 30 29 28 27 26
118412024-02-08T06:43:55.419Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:54.643313258Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1570}
118422024-02-08T06:43:55.419ZWrote out file "/tmp/test_repair_verify.data"
118432024-02-08T06:43:55.422Z
118442024-02-08T06:43:55.423ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
118452024-02-08T06:43:55.436ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118462024-02-08T06:43:55.436Z 0 000-019 95 95 95 153 153 153 F F F
118472024-02-08T06:43:55.436Z 1 020-039 100 100 100 161 161 161 F F F
118482024-02-08T06:43:55.436Z 2 040-059 104 104 104 168 168 168 F F F
118492024-02-08T06:43:55.436Z 3 060-079 105 105 105 170 170 170 F F F
118502024-02-08T06:43:55.436Z 4 080-099 105 105 105 170 170 170 F F F
118512024-02-08T06:43:55.436Z 5 100-119 104 104 104 168 168 168 F F F
118522024-02-08T06:43:55.436Z 6 120-139 100 100 100 161 161 161 F F F
118532024-02-08T06:43:55.436Z 7 140-159 104 104 104 169 169 169 F F F
118542024-02-08T06:43:55.436Z 8 160-179 107 107 107 172 172 172 F F F
118552024-02-08T06:43:55.436Z 9 180-199 106 108 108 171 173 173 F F F <---
118562024-02-08T06:43:55.436Z 10 200-219 104 108 108 169 174 174 F F F <---
118572024-02-08T06:43:55.440Z 11 220-239 94 94 94 152 152 152 F F F
118582024-02-08T06:43:55.440Z 12 240-259 107 107 107 172 172 172 F F F
118592024-02-08T06:43:55.440Z 13 260-279 105 108 108 170 173 173 F F F <---
118602024-02-08T06:43:55.440Z 14 280-299 105 108 108 170 173 173 F F F <---
118612024-02-08T06:43:55.440Z 15 300-319 107 107 107 172 172 172 F T T <---
118622024-02-08T06:43:55.440Z 16 320-339 106 106 106 171 171 171 F F F
118632024-02-08T06:43:55.440Z 17 340-359 103 103 103 167 167 167 F F F
118642024-02-08T06:43:55.440Z 18 360-379 104 104 104 169 169 169 F F F
118652024-02-08T06:43:55.440Z 19 380-399 106 106 106 171 171 171 F F F
118662024-02-08T06:43:55.440Z 20 400-419 106 106 106 171 171 171 F F F
118672024-02-08T06:43:55.440Z 21 420-439 107 107 107 172 172 172 F F F
118682024-02-08T06:43:55.440Z 22 440-459 107 108 108 172 174 174 F F F <---
118692024-02-08T06:43:55.440Z 23 460-479 101 101 101 162 162 162 F F F
118702024-02-08T06:43:55.441Z 24 480-499 107 107 107 172 172 172 F F F
118712024-02-08T06:43:55.441Z 25 500-519 102 102 102 163 163 163 F F F
118722024-02-08T06:43:55.441Z 26 520-539 104 104 104 168 168 168 F T T <---
118732024-02-08T06:43:55.441Z 27 540-559 103 103 103 167 167 167 F F F
118742024-02-08T06:43:55.441Z 28 560-579 107 108 108 172 174 174 F F F <---
118752024-02-08T06:43:55.441Z 29 580-599 105 108 108 170 173 173 F F F <---
118762024-02-08T06:43:55.441ZMax gen: 108, Max flush: 174
118772024-02-08T06:43:55.441ZError: Difference in extent metadata found!
118782024-02-08T06:43:55.441ZA Difference in extent metadata is expected here
118792024-02-08T06:43:55.441ZOn loop 54
118802024-02-08T06:43:55.441Z
118812024-02-08T06:43:55.444ZVerifying data now
118822024-02-08T06:43:59.928ZNo change after verify
118832024-02-08T06:43:59.929ZLoop: 54 Downstairs dump after verify (and repair):
118842024-02-08T06:43:59.933ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118852024-02-08T06:43:59.933Z 0 000-019 95 95 95 153 153 153 F F F
118862024-02-08T06:43:59.933Z 1 020-039 100 100 100 161 161 161 F F F
118872024-02-08T06:43:59.933Z 2 040-059 104 104 104 168 168 168 F F F
118882024-02-08T06:43:59.933Z 3 060-079 105 105 105 170 170 170 F F F
118892024-02-08T06:43:59.933Z 4 080-099 105 105 105 170 170 170 F F F
118902024-02-08T06:43:59.933Z 5 100-119 104 104 104 168 168 168 F F F
118912024-02-08T06:43:59.933Z 6 120-139 100 100 100 161 161 161 F F F
118922024-02-08T06:43:59.933Z 7 140-159 104 104 104 169 169 169 F F F
118932024-02-08T06:43:59.933Z 8 160-179 107 107 107 172 172 172 F F F
118942024-02-08T06:43:59.933Z 9 180-199 108 108 108 173 173 173 F F F
118952024-02-08T06:43:59.933Z 10 200-219 108 108 108 174 174 174 F F F
118962024-02-08T06:43:59.933Z 11 220-239 94 94 94 152 152 152 F F F
118972024-02-08T06:43:59.934Z 12 240-259 107 107 107 172 172 172 F F F
118982024-02-08T06:43:59.934Z 13 260-279 108 108 108 173 173 173 F F F
118992024-02-08T06:43:59.934Z 14 280-299 108 108 108 173 173 173 F F F
119002024-02-08T06:43:59.934Z 15 300-319 109 109 109 175 175 175 F F F
119012024-02-08T06:43:59.934Z 16 320-339 106 106 106 171 171 171 F F F
119022024-02-08T06:43:59.934Z 17 340-359 103 103 103 167 167 167 F F F
119032024-02-08T06:43:59.934Z 18 360-379 104 104 104 169 169 169 F F F
119042024-02-08T06:43:59.934Z 19 380-399 106 106 106 171 171 171 F F F
119052024-02-08T06:43:59.934Z 20 400-419 106 106 106 171 171 171 F F F
119062024-02-08T06:43:59.934Z 21 420-439 107 107 107 172 172 172 F F F
119072024-02-08T06:43:59.934Z 22 440-459 108 108 108 174 174 174 F F F
119082024-02-08T06:43:59.934Z 23 460-479 101 101 101 162 162 162 F F F
119092024-02-08T06:43:59.934Z 24 480-499 107 107 107 172 172 172 F F F
119102024-02-08T06:43:59.934Z 25 500-519 102 102 102 163 163 163 F F F
119112024-02-08T06:43:59.934Z 26 520-539 109 109 109 175 175 175 F F F
119122024-02-08T06:43:59.934Z 27 540-559 103 103 103 167 167 167 F F F
119132024-02-08T06:43:59.935Z 28 560-579 108 108 108 174 174 174 F F F
119142024-02-08T06:43:59.935Z 29 580-599 108 108 108 173 173 173 F F F
119152024-02-08T06:43:59.935ZMax gen: 109, Max flush: 175
119162024-02-08T06:43:59.935Z
119172024-02-08T06:43:59.935ZBegin loop 55
119182024-02-08T06:43:59.935ZDownstairs to restart: 2
119192024-02-08T06:43:59.944Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.170424473Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578}
119202024-02-08T06:43:59.944Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.170663495Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578}
119212024-02-08T06:43:59.950Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","levelCrucible runtime is spawned
119222024-02-08T06:43:59.950Z":30,"time":"2024-02-08T06:43:59.170706251Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578}
119232024-02-08T06:43:59.950Z{"msg":"Crucible 285a5c6e-b37d-40ae-9e4c-517fc2e75740 has session id: ee62d24c-f701-439a-8ad6-3868532de148","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.17073143Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119242024-02-08T06:43:59.950Z{"msg":"Upstairs opts: Upstairs UUID: 285a5c6e-b37d-40ae-9e4c-517fc2e75740, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.170782516Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119252024-02-08T06:43:59.950Z{"msg":"Crucible stats registered with UUID: 285a5c6e-b37d-40ae-9e4c-517fc2e75740","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.170811144Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119262024-02-08T06:43:59.950Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.170833812Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578}
119272024-02-08T06:43:59.950Z{"msg":"285a5c6e-b37d-40ae-9e4c-517fc2e75740 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.170927465Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119282024-02-08T06:43:59.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.170953593Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119292024-02-08T06:43:59.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.170974422Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119302024-02-08T06:43:59.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171003229Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119312024-02-08T06:43:59.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171017988Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119322024-02-08T06:43:59.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171040966Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119332024-02-08T06:43:59.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171056355Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119342024-02-08T06:43:59.951Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171069774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119352024-02-08T06:43:59.951Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171142579Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119362024-02-08T06:43:59.951Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171181256Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"io task","client":"1","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119372024-02-08T06:43:59.951Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:59.171273399Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119382024-02-08T06:43:59.951Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:59.1713974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119392024-02-08T06:43:59.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171441366Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"io task","client":"1","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119402024-02-08T06:43:59.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171472634Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119412024-02-08T06:43:59.951Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:43:59.171514701Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119422024-02-08T06:43:59.951Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171541999Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119432024-02-08T06:43:59.951Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171576936Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119442024-02-08T06:43:59.951Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171608914Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119452024-02-08T06:43:59.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.171983376Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119462024-02-08T06:43:59.951Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.172035022Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119472024-02-08T06:43:59.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.172053161Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119482024-02-08T06:43:59.951Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.172098937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119492024-02-08T06:43:59.952Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.172630347Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119502024-02-08T06:43:59.952Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.172666714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119512024-02-08T06:43:59.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.173231602Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119522024-02-08T06:43:59.952Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.173356113Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119532024-02-08T06:43:59.952Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.1733906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119542024-02-08T06:43:59.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.173404719Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119552024-02-08T06:43:59.952Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.173425717Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119562024-02-08T06:43:59.952Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:43:59.173440196Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119572024-02-08T06:44:00.943Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.172039804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119582024-02-08T06:44:00.949Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.17221405Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119592024-02-08T06:44:00.949Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.173038069Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119602024-02-08T06:44:00.949Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.173075026Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119612024-02-08T06:44:00.949Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.173595037Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119622024-02-08T06:44:00.949Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174148925Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119632024-02-08T06:44:00.949Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174197111Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119642024-02-08T06:44:00.949Z{"msg":"[0]R flush_numbers[0..12]: [153, 161, 168, 170, 170, 168, 161, 169, 172, 173, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.17421863Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119652024-02-08T06:44:00.950Z{"msg":"[0]R generation[0..12]: [95, 100, 104, 105, 105, 104, 100, 104, 107, 108, 108, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174247738Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119662024-02-08T06:44:00.950Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174268646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119672024-02-08T06:44:00.950Z{"msg":"[1]R flush_numbers[0..12]: [153, 161, 168, 170, 170, 168, 161, 169, 172, 173, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174293474Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119682024-02-08T06:44:00.950Z{"msg":"[1]R generation[0..12]: [95, 100, 104, 105, 105, 104, 100, 104, 107, 108, 108, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174307743Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119692024-02-08T06:44:00.950Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174329641Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119702024-02-08T06:44:00.950Z{"msg":"[2]R flush_numbers[0..12]: [153, 161, 168, 170, 170, 168, 161, 169, 172, 173, 174, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.17434505Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119712024-02-08T06:44:00.950Z{"msg":"[2]R generation[0..12]: [95, 100, 104, 105, 105, 104, 100, 104, 107, 108, 108, 94]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
119722024-02-08T06:44:00.950Z,"time":"2024-02-08T06:44:00.174377678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119732024-02-08T06:44:00.950Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174416255Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119742024-02-08T06:44:00.950Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174450292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119752024-02-08T06:44:00.950Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.17447559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119762024-02-08T06:44:00.950Z{"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174505998Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119772024-02-08T06:44:00.950Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174542156Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119782024-02-08T06:44:00.950Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174569623Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119792024-02-08T06:44:00.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174583773Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119802024-02-08T06:44:00.951Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.17461496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119812024-02-08T06:44:00.951Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174651777Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119822024-02-08T06:44:00.951Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174678445Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119832024-02-08T06:44:00.951Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174695124Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119842024-02-08T06:44:00.951Z{"msg":"285a5c6e-b37d-40ae-9e4c-517fc2e75740 is now active with session: ee62d24c-f701-439a-8ad6-3868532de148","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174730202Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119852024-02-08T06:44:00.951Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174762119Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578,"session_id":"ee62d24c-f701-439a-8ad6-3868532de148"}
119862024-02-08T06:44:00.951Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:00.174777148Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578}
119872024-02-08T06:44:00.951ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
119882024-02-08T06:44:00.951ZLoading write count information from file "/tmp/test_repair_verify.data"
119892024-02-08T06:44:00.951ZRun Repair workload
119902024-02-08T06:44:00.951Z01/30 Write block 24 len 20480 data: 12 13 12 12 12
119912024-02-08T06:44:00.955Z02/30 Read block 185 len 12288
119922024-02-08T06:44:00.955Z03/30 Write block 291 len 8192 data: 10 8
119932024-02-08T06:44:00.955Z04/30 Read block 147 len 4096
119942024-02-08T06:44:00.955Z05/30 Write block 557 len 24576 data: 9 6 6 8 8 8
119952024-02-08T06:44:00.955Z06/30 Read block 332 len 40960
119962024-02-08T06:44:00.955Z07/30 Read block 257 len 20480
119972024-02-08T06:44:00.955Z08/30 Read block 506 len 12288
119982024-02-08T06:44:00.955Z09/30 Flush
119992024-02-08T06:44:00.958Z10/30 Flush
120002024-02-08T06:44:00.958Z11/30 Write block 239 len 20480 data: 7 6 7 6 10
120012024-02-08T06:44:00.958Z12/30 Write block 423 len 40960 data: 8 7 6 6 4 4 5 7 6 7
120022024-02-08T06:44:00.959Z13/30 Read block 268 len 36864
120032024-02-08T06:44:00.962Z14/30 Read block 447 len 20480
120042024-02-08T06:44:00.962Z15/30 Write block 393 len 20480 data: 7 7 6 4 6
120052024-02-08T06:44:00.962Z16/30 Read block 460 len 36864
120062024-02-08T06:44:00.962Z17/30 Read block 197 len 12288
120072024-02-08T06:44:00.965Z18/30 Flush
120082024-02-08T06:44:00.965Z19/30 Flush
120092024-02-08T06:44:00.968Z20/30 Write block 507 len 40960 data: 5 5 7 9 6 5 6 5 5 7
120102024-02-08T06:44:00.968Z21/30 Read block 41 len 20480
120112024-02-08T06:44:00.968Z22/30 Flush
120122024-02-08T06:44:00.972Z23/30 Write block 61 len 36864 data: 5 6 5 4 4 3 4 8 8
120132024-02-08T06:44:00.972Z24/30 Write block 129 len 8192 data: 6 6
120142024-02-08T06:44:00.972Z25/30 Read block 479 len 36864
120152024-02-08T06:44:00.972Z26/30 Write block 238 len 24576 data: 7 8 7 8 7 11
120162024-02-08T06:44:00.976Z27/30 Write block 492 len 40960 data: 10 11 9 8 6 6 8 9 10 8
120172024-02-08T06:44:00.976Z28/30 Write block 517 len 40960 data: 8 9 10 8 6 6 7 7 7 7
120182024-02-08T06:44:00.976Z29/30 Write block 62 len 8192 data: 7 6
120192024-02-08T06:44:00.977Z30/30 Write block 39 len 12288 data: 6 6 6
120202024-02-08T06:44:00.977Z----------------------------------------------------------------
120212024-02-08T06:44:00.977Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
120222024-02-08T06:44:00.977ZStates: Active Active Active
120232024-02-08T06:44:00.977ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
120242024-02-08T06:44:00.977Z 1 Acked 1000 Write 5 Done Done Done false
120252024-02-08T06:44:00.977Z 2 Acked 1001 Read 3 Done Done Done false
120262024-02-08T06:44:00.977Z 3 Acked 1002 Write 2 Done Done Done false
120272024-02-08T06:44:00.977Z 4 Acked 1003 Read 1 Done Done Done false
120282024-02-08T06:44:00.977Z 5 Acked 1004 Write 6 Done Done Done false
120292024-02-08T06:44:00.977Z 6 Acked 1005 Read 10 Done Done Done false
120302024-02-08T06:44:00.977Z 7 Acked 1006 Read 5 Done Done Done false
120312024-02-08T06:44:00.977Z 8 Acked 1007 Read 3 Done Done Done false
120322024-02-08T06:44:00.977Z 9 Acked 1008 Flush 0 Done Done Sent false
120332024-02-08T06:44:00.977Z 10 Acked 1009 Flush 0 Done Done Sent false
120342024-02-08T06:44:00.978Z 11 Acked 1010 Write 5 Done Done Sent false
120352024-02-08T06:44:00.978Z 12 Acked 1011 Write 10 Done Done Sent false
120362024-02-08T06:44:00.978Z 13 Acked 1012 Read 9 Done Done Sent false
120372024-02-08T06:44:00.978Z 14 Acked 1013 Read 5 Done Done Sent false
120382024-02-08T06:44:00.978Z 15 Acked 1014 Write 5 Done Done Sent false
120392024-02-08T06:44:00.978Z 16 Acked 1015 Read 9 Done Done Sent false
120402024-02-08T06:44:00.978Z 17 Acked 1016 Read 3 Done Done Sent false
120412024-02-08T06:44:00.978Z 18 Acked 1017 Flush 0 Done Done Sent false
120422024-02-08T06:44:00.978Z 19 Acked 1018 Flush 0 Done Done Sent false
120432024-02-08T06:44:00.978Z 20 Acked 1019 Write 10 Done Done Sent false
120442024-02-08T06:44:00.978Z 21 Acked 1020 Read 5 Done Done Sent false
120452024-02-08T06:44:00.978Z 22 Acked 1021 Flush 0 Done Done Sent false
120462024-02-08T06:44:00.978Z 23 Acked 1022 Write 9 Done Done Sent false
120472024-02-08T06:44:00.978Z 24 Acked 1023 Write 2 Done Done Sent false
120482024-02-08T06:44:00.978Z 25 Acked 1024 Read 9 Done Done Sent false
120492024-02-08T06:44:00.979Z 26 Acked 1025 Write 6 Done Done Sent false
120502024-02-08T06:44:00.979Z 27 Acked 1026 Write 10 Done Done Sent false
120512024-02-08T06:44:00.979Z 28 Acked 1027 Write 10 Done Done Sent false
120522024-02-08T06:44:00.979Z 29 Acked 1028 Write 2 Sent Sent Sent false
120532024-02-08T06:44:00.979Z 30 Acked 1029 Write 3 Sent Sent Sent false
120542024-02-08T06:44:00.979Z STATES DS:0 DS:1 DS:2 TOTAL
120552024-02-08T06:44:00.979Z New 0 0 0 0
120562024-02-08T06:44:00.979Z Sent 2 2 22 26
120572024-02-08T06:44:00.979Z Done 28 28 8 64
120582024-02-08T06:44:00.979Z Skipped 0 0 0 0
120592024-02-08T06:44:00.979Z Error 0 0 0 0
120602024-02-08T06:44:00.979ZLast Flush: 1021 1021 0
120612024-02-08T06:44:00.979ZDownstairs last five completed:
120622024-02-08T06:44:00.979ZUpstairs last five completed: 30 29 28 27 26
120632024-02-08T06:44:00.983Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:00.204657704Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1578}
120642024-02-08T06:44:00.983ZWrote out file "/tmp/test_repair_verify.data"
120652024-02-08T06:44:00.983Z
120662024-02-08T06:44:00.986ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
120672024-02-08T06:44:01.003ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120682024-02-08T06:44:01.003Z 0 000-019 95 95 95 153 153 153 F F F
120692024-02-08T06:44:01.003Z 1 020-039 110 110 100 176 176 161 T T T <---
120702024-02-08T06:44:01.003Z 2 040-059 104 104 104 168 168 168 T T F <---
120712024-02-08T06:44:01.009Z 3 060-079 105 105 105 170 170 170 T T F <---
120722024-02-08T06:44:01.009Z 4 080-099 105 105 105 170 170 170 F F F
120732024-02-08T06:44:01.010Z 5 100-119 104 104 104 168 168 168 F F F
120742024-02-08T06:44:01.010Z 6 120-139 100 100 100 161 161 161 T T F <---
120752024-02-08T06:44:01.010Z 7 140-159 104 104 104 169 169 169 F F F
120762024-02-08T06:44:01.010Z 8 160-179 107 107 107 172 172 172 F F F
120772024-02-08T06:44:01.010Z 9 180-199 108 108 108 173 173 173 F F F
120782024-02-08T06:44:01.010Z 10 200-219 108 108 108 174 174 174 F F F
120792024-02-08T06:44:01.010Z 11 220-239 110 110 94 178 178 152 T T F <---
120802024-02-08T06:44:01.010Z 12 240-259 110 110 107 178 178 172 T T F <---
120812024-02-08T06:44:01.010Z 13 260-279 108 108 108 173 173 173 F F F
120822024-02-08T06:44:01.010Z 14 280-299 110 110 108 176 176 173 F F T <---
120832024-02-08T06:44:01.010Z 15 300-319 109 109 109 175 175 175 F F F
120842024-02-08T06:44:01.010Z 16 320-339 106 106 106 171 171 171 F F F
120852024-02-08T06:44:01.010Z 17 340-359 103 103 103 167 167 167 F F F
120862024-02-08T06:44:01.010Z 18 360-379 104 104 104 169 169 169 F F F
120872024-02-08T06:44:01.010Z 19 380-399 110 110 106 178 178 171 F F F <---
120882024-02-08T06:44:01.011Z 20 400-419 106 106 106 171 171 171 F F F
120892024-02-08T06:44:01.011Z 21 420-439 110 110 107 178 178 172 F F F <---
120902024-02-08T06:44:01.011Z 22 440-459 108 108 108 174 174 174 F F F
120912024-02-08T06:44:01.011Z 23 460-479 101 101 101 162 162 162 F F F
120922024-02-08T06:44:01.011Z 24 480-499 107 107 107 172 172 172 T T F <---
120932024-02-08T06:44:01.011Z 25 500-519 110 110 102 180 180 163 T T F <---
120942024-02-08T06:44:01.011Z 26 520-539 109 109 109 175 175 175 T T F <---
120952024-02-08T06:44:01.011Z 27 540-559 110 110 103 176 176 167 F F T <---
120962024-02-08T06:44:01.011Z 28 560-579 110 110 108 176 176 174 F F T <---
120972024-02-08T06:44:01.011Z 29 580-599 108 108 108 173 173 173 F F F
120982024-02-08T06:44:01.011ZMax gen: 110, Max flush: 180
120992024-02-08T06:44:01.011ZError: Difference in extent metadata found!
121002024-02-08T06:44:01.011ZA Difference in extent metadata is expected here
121012024-02-08T06:44:01.011ZOn loop 55
121022024-02-08T06:44:01.011Z
121032024-02-08T06:44:01.015ZVerifying data now
121042024-02-08T06:44:06.698ZNo change after verify
121052024-02-08T06:44:06.698ZLoop: 55 Downstairs dump after verify (and repair):
121062024-02-08T06:44:06.714ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121072024-02-08T06:44:06.714Z 0 000-019 95 95 95 153 153 153 F F F
121082024-02-08T06:44:06.717Z 1 020-039 111 111 111 181 181 181 F F F
121092024-02-08T06:44:06.718Z 2 040-059 111 111 111 181 181 181 F F F
121102024-02-08T06:44:06.718Z 3 060-079 111 111 111 181 181 181 F F F
121112024-02-08T06:44:06.718Z 4 080-099 105 105 105 170 170 170 F F F
121122024-02-08T06:44:06.718Z 5 100-119 104 104 104 168 168 168 F F F
121132024-02-08T06:44:06.718Z 6 120-139 111 111 111 181 181 181 F F F
121142024-02-08T06:44:06.718Z 7 140-159 104 104 104 169 169 169 F F F
121152024-02-08T06:44:06.718Z 8 160-179 107 107 107 172 172 172 F F F
121162024-02-08T06:44:06.718Z 9 180-199 108 108 108 173 173 173 F F F
121172024-02-08T06:44:06.718Z 10 200-219 108 108 108 174 174 174 F F F
121182024-02-08T06:44:06.718Z 11 220-239 111 111 111 181 181 181 F F F
121192024-02-08T06:44:06.718Z 12 240-259 111 111 111 181 181 181 F F F
121202024-02-08T06:44:06.718Z 13 260-279 108 108 108 173 173 173 F F F
121212024-02-08T06:44:06.718Z 14 280-299 110 110 110 176 176 176 F F F
121222024-02-08T06:44:06.718Z 15 300-319 109 109 109 175 175 175 F F F
121232024-02-08T06:44:06.719Z 16 320-339 106 106 106 171 171 171 F F F
121242024-02-08T06:44:06.719Z 17 340-359 103 103 103 167 167 167 F F F
121252024-02-08T06:44:06.719Z 18 360-379 104 104 104 169 169 169 F F F
121262024-02-08T06:44:06.719Z 19 380-399 110 110 110 178 178 178 F F F
121272024-02-08T06:44:06.719Z 20 400-419 106 106 106 171 171 171 F F F
121282024-02-08T06:44:06.719Z 21 420-439 110 110 110 178 178 178 F F F
121292024-02-08T06:44:06.719Z 22 440-459 108 108 108 174 174 174 F F F
121302024-02-08T06:44:06.719Z 23 460-479 101 101 101 162 162 162 F F F
121312024-02-08T06:44:06.719Z 24 480-499 111 111 111 181 181 181 F F F
121322024-02-08T06:44:06.719Z 25 500-519 111 111 111 181 181 181 F F F
121332024-02-08T06:44:06.719Z 26 520-539 111 111 111 181 181 181 F F F
121342024-02-08T06:44:06.719Z 27 540-559 110 110 110 176 176 176 F F F
121352024-02-08T06:44:06.719Z 28 560-579 110 110 110 176 176 176 F F F
121362024-02-08T06:44:06.719Z 29 580-599 108 108 108 173 173 173 F F F
121372024-02-08T06:44:06.719ZMax gen: 111, Max flush: 181
121382024-02-08T06:44:06.719Z
121392024-02-08T06:44:06.720ZBegin loop 56
121402024-02-08T06:44:06.720ZDownstairs to restart: 1
121412024-02-08T06:44:06.736Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.964773726Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586}
121422024-02-08T06:44:06.736Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965054555Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586}
121432024-02-08T06:44:06.736ZCrucible runtime is spawned
121442024-02-08T06:44:06.743Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965100702Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586}
121452024-02-08T06:44:06.743Z{"msg":"Crucible 258f64e9-148f-47cf-b9c9-c72b6f92b484 has session id: a3650dd5-ab26-4c99-b60c-4e4af8368592","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965134819Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121462024-02-08T06:44:06.743Z{"msg":"Upstairs opts: Upstairs UUID: 258f64e9-148f-47cf-b9c9-c72b6f92b484, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965177976Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121472024-02-08T06:44:06.743Z{"msg":"Crucible stats registered with UUID: 258f64e9-148f-47cf-b9c9-c72b6f92b484","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965203824Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121482024-02-08T06:44:06.743Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965237882Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586}
121492024-02-08T06:44:06.743Z{"msg":"258f64e9-148f-47cf-b9c9-c72b6f92b484 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965267509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121502024-02-08T06:44:06.743Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965303137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121512024-02-08T06:44:06.743Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965352193Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121522024-02-08T06:44:06.743Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965379031Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121532024-02-08T06:44:06.743Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965415318Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121542024-02-08T06:44:06.743Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965454515Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121552024-02-08T06:44:06.743Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965505402Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121562024-02-08T06:44:06.743Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965545148Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121572024-02-08T06:44:06.743Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965584856Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121582024-02-08T06:44:06.744Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965612754Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121592024-02-08T06:44:06.744Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:05.965642701Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121602024-02-08T06:44:06.744Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965680158Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121612024-02-08T06:44:06.744Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:05.965708156Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121622024-02-08T06:44:06.744Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965743444Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121632024-02-08T06:44:06.744Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:05.96578605Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121642024-02-08T06:44:06.744Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965825847Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121652024-02-08T06:44:06.744Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965852425Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121662024-02-08T06:44:06.744Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.965986305Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121672024-02-08T06:44:06.744Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.966389505Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121682024-02-08T06:44:06.744Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.966436292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121692024-02-08T06:44:06.744Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.966469119Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121702024-02-08T06:44:06.744Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.966508336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121712024-02-08T06:44:06.744Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.967051955Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121722024-02-08T06:44:06.744Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.967187095Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121732024-02-08T06:44:06.744Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.967572326Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121742024-02-08T06:44:06.745Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.967616273Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121752024-02-08T06:44:06.745Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.967647081Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121762024-02-08T06:44:06.745Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.967666969Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121772024-02-08T06:44:06.745Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.967681638Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121782024-02-08T06:44:06.745Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:05.967704006Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121792024-02-08T06:44:07.738Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.966422205Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121802024-02-08T06:44:07.743Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.96661131Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121812024-02-08T06:44:07.743Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.967360844Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121822024-02-08T06:44:07.743Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.967393342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121832024-02-08T06:44:07.744Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.967915232Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121842024-02-08T06:44:07.744Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968495259Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121852024-02-08T06:44:07.744Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968544435Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121862024-02-08T06:44:07.744Z{"msg":"[0]R flush_numbers[0..12]: [153, 181, 181, 181, 170, 168, 181, 169, 172, 173, 174, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968582693Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121872024-02-08T06:44:07.744Z{"msg":"[0]R generation[0..12]: [95, 111, 111, 111, 105, 104, 111, 104, 107, 108, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.96861968Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121882024-02-08T06:44:07.744Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968656797Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121892024-02-08T06:44:07.744Z{"msg":"[1]R flush_numbers[0..12]: [153, 181, 181, 181, 170, 168, 181, 169, 172, 173, 174, 181]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
121902024-02-08T06:44:07.744Z,"time":"2024-02-08T06:44:06.968692984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121912024-02-08T06:44:07.744Z{"msg":"[1]R generation[0..12]: [95, 111, 111, 111, 105, 104, 111, 104, 107, 108, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968735051Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121922024-02-08T06:44:07.744Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968775088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121932024-02-08T06:44:07.744Z{"msg":"[2]R flush_numbers[0..12]: [153, 181, 181, 181, 170, 168, 181, 169, 172, 173, 174, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968823614Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121942024-02-08T06:44:07.744Z{"msg":"[2]R generation[0..12]: [95, 111, 111, 111, 105, 104, 111, 104, 107, 108, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968854942Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121952024-02-08T06:44:07.744Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968903258Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121962024-02-08T06:44:07.744Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968941466Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121972024-02-08T06:44:07.744Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.968975783Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121982024-02-08T06:44:07.745Z{"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.96900725Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
121992024-02-08T06:44:07.745Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.969040088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
122002024-02-08T06:44:07.745Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.969060937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
122012024-02-08T06:44:07.745ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
122022024-02-08T06:44:07.745Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.969091974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
122032024-02-08T06:44:07.745Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.969121622Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
122042024-02-08T06:44:07.745Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.969154259Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
122052024-02-08T06:44:07.745Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.969184897Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
122062024-02-08T06:44:07.745Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.969215135Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
122072024-02-08T06:44:07.745Z{"msg":"258f64e9-148f-47cf-b9c9-c72b6f92b484 is now active with session: a3650dd5-ab26-4c99-b60c-4e4af8368592","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.969247023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
122082024-02-08T06:44:07.745Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.96927679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586,"session_id":"a3650dd5-ab26-4c99-b60c-4e4af8368592"}
122092024-02-08T06:44:07.745Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:06.969305748Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586}
122102024-02-08T06:44:07.745ZLoading write count information from file "/tmp/test_repair_verify.data"
122112024-02-08T06:44:07.745ZRun Repair workload
122122024-02-08T06:44:07.745Z01/30 Write block 559 len 24576 data: 7 9 9 9 8 7
122132024-02-08T06:44:07.749Z02/30 Read block 466 len 36864
122142024-02-08T06:44:07.749Z03/30 Read block 98 len 40960
122152024-02-08T06:44:07.749Z04/30 Read block 317 len 28672
122162024-02-08T06:44:07.749Z05/30 Write block 454 len 20480 data: 8 11 10 10 11
122172024-02-08T06:44:07.749Z06/30 Read block 478 len 32768
122182024-02-08T06:44:07.749Z07/30 Read block 59 len 32768
122192024-02-08T06:44:07.753Z08/30 Flush
122202024-02-08T06:44:07.753Z09/30 Write block 81 len 8192 data: 6 7
122212024-02-08T06:44:07.753Z10/30 Read block 337 len 24576
122222024-02-08T06:44:07.756Z11/30 Flush
122232024-02-08T06:44:07.756Z12/30 Write block 472 len 28672 data: 9 8 9 9 6 5 5
122242024-02-08T06:44:07.756Z13/30 Write block 75 len 28672 data: 7 8 5 6 7 8 7
122252024-02-08T06:44:07.756Z14/30 Read block 375 len 32768
122262024-02-08T06:44:07.763Z15/30 Write block 532 len 40960 data: 13 12 12 12 12 10 8 6 8 7
122272024-02-08T06:44:07.763Z16/30 Write block 380 len 4096 data: 6
122282024-02-08T06:44:07.763Z17/30 Read block 155 len 40960
122292024-02-08T06:44:07.763Z18/30 Flush
122302024-02-08T06:44:07.766Z19/30 Write block 461 len 24576 data: 9 10 10 8 7 8
122312024-02-08T06:44:07.766Z20/30 Write block 565 len 4096 data: 8
122322024-02-08T06:44:07.766Z21/30 Read block 91 len 4096
122332024-02-08T06:44:07.766Z22/30 Write block 296 len 28672 data: 5 4 7 10 12 14 15
122342024-02-08T06:44:07.766Z23/30 Read block 567 len 24576
122352024-02-08T06:44:07.769Z24/30 Read block 203 len 36864
122362024-02-08T06:44:07.770Z25/30 Write block 424 len 8192 data: 8 7
122372024-02-08T06:44:07.770Z26/30 Write block 290 len 20480 data: 10 11 9 7 5
122382024-02-08T06:44:07.770Z27/30 Write block 543 len 4096 data: 7
122392024-02-08T06:44:07.770Z28/30 Read block 522 len 28672
122402024-02-08T06:44:07.776Z29/30 Write block 414 len 32768 data: 7 7 6 8 9 9 9 8
122412024-02-08T06:44:07.776Z30/30 Write block 341 len 40960 data: 12 11 10 9 10 10 8 6 6 7
122422024-02-08T06:44:07.776Z----------------------------------------------------------------
122432024-02-08T06:44:07.776Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
122442024-02-08T06:44:07.776ZStates: Active Active Active
122452024-02-08T06:44:07.777ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
122462024-02-08T06:44:07.777Z 1 Acked 1000 Write 6 Done Sent Done false
122472024-02-08T06:44:07.777Z 2 Acked 1001 Read 9 Done Sent Done false
122482024-02-08T06:44:07.777Z 3 Acked 1002 Read 10 Done Sent Done false
122492024-02-08T06:44:07.777Z 4 Acked 1003 Read 7 Done Sent Done false
122502024-02-08T06:44:07.777Z 5 Acked 1004 Write 5 Done Sent Done false
122512024-02-08T06:44:07.777Z 6 Acked 1005 Read 8 Done Sent Done false
122522024-02-08T06:44:07.777Z 7 Acked 1006 Read 8 Done Sent Done false
122532024-02-08T06:44:07.777Z 8 Acked 1007 Flush 0 Done Sent Done false
122542024-02-08T06:44:07.777Z 9 Acked 1008 Write 2 Done Sent Done false
122552024-02-08T06:44:07.777Z 10 Acked 1009 Read 6 Done Sent Done false
122562024-02-08T06:44:07.777Z 11 Acked 1010 Flush 0 Done Sent Done false
122572024-02-08T06:44:07.777Z 12 Acked 1011 Write 7 Done Sent Done false
122582024-02-08T06:44:07.777Z 13 Acked 1012 Write 7 Done Sent Done false
122592024-02-08T06:44:07.778Z 14 Acked 1013 Read 8 Done Sent Done false
122602024-02-08T06:44:07.778Z 15 Acked 1014 Write 10 Done Sent Done false
122612024-02-08T06:44:07.778Z 16 Acked 1015 Write 1 Done Sent Done false
122622024-02-08T06:44:07.778Z 17 Acked 1016 Read 10 Done Sent Done false
122632024-02-08T06:44:07.778Z 18 Acked 1017 Flush 0 Done Sent Done false
122642024-02-08T06:44:07.778Z 19 Acked 1018 Write 6 Done Sent Done false
122652024-02-08T06:44:07.778Z 20 Acked 1019 Write 1 Done Sent Done false
122662024-02-08T06:44:07.778Z 21 Acked 1020 Read 1 Done Sent Done false
122672024-02-08T06:44:07.778Z 22 Acked 1021 Write 7 Done Sent Done false
122682024-02-08T06:44:07.778Z 23 Acked 1022 Read 6 Done Sent Done false
122692024-02-08T06:44:07.778Z 24 Acked 1023 Read 9 Done Sent Done false
122702024-02-08T06:44:07.778Z 25 Acked 1024 Write 2 Done Sent Done false
122712024-02-08T06:44:07.778Z 26 Acked 1025 Write 5 Done Sent Done false
122722024-02-08T06:44:07.778Z 27 Acked 1026 Write 1 Done Sent Done false
122732024-02-08T06:44:07.778Z 28 Acked 1027 Read 7 Done Sent Done false
122742024-02-08T06:44:07.778Z 29 Acked 1028 Write 8 Done Sent Done false
122752024-02-08T06:44:07.779Z 30 Acked 1029 Write 10 Sent Sent Sent false
122762024-02-08T06:44:07.779Z STATES DS:0 DS:1 DS:2 TOTAL
122772024-02-08T06:44:07.779Z New 0 0 0 0
122782024-02-08T06:44:07.779Z Sent 1 30 1 32
122792024-02-08T06:44:07.779Z Done 29 0 29 58
122802024-02-08T06:44:07.779Z Skipped 0 0 0 0
122812024-02-08T06:44:07.779Z Error 0 0 0 0
122822024-02-08T06:44:07.779ZLast Flush: 1017 0 1017
122832024-02-08T06:44:07.779ZDownstairs last five completed:
122842024-02-08T06:44:07.779ZUpstairs last five completed: 30 29 28 27 26
122852024-02-08T06:44:07.779Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:07.00098606Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1586}
122862024-02-08T06:44:07.779ZWrote out file "/tmp/test_repair_verify.data"
122872024-02-08T06:44:07.783Z
122882024-02-08T06:44:07.783ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
122892024-02-08T06:44:07.799ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122902024-02-08T06:44:07.799Z 0 000-019 95 95 95 153 153 153 F F F
122912024-02-08T06:44:07.799Z 1 020-039 111 111 111 181 181 181 F F F
122922024-02-08T06:44:07.799Z 2 040-059 111 111 111 181 181 181 F F F
122932024-02-08T06:44:07.799Z 3 060-079 112 111 112 184 181 184 F F F <---
122942024-02-08T06:44:07.799Z 4 080-099 112 105 112 184 170 184 F F F <---
122952024-02-08T06:44:07.799Z 5 100-119 104 104 104 168 168 168 F F F
122962024-02-08T06:44:07.806Z 6 120-139 111 111 111 181 181 181 F F F
122972024-02-08T06:44:07.806Z 7 140-159 104 104 104 169 169 169 F F F
122982024-02-08T06:44:07.806Z 8 160-179 107 107 107 172 172 172 F F F
122992024-02-08T06:44:07.807Z 9 180-199 108 108 108 173 173 173 F F F
123002024-02-08T06:44:07.807Z 10 200-219 108 108 108 174 174 174 F F F
123012024-02-08T06:44:07.807Z 11 220-239 111 111 111 181 181 181 F F F
123022024-02-08T06:44:07.807Z 12 240-259 111 111 111 181 181 181 F F F
123032024-02-08T06:44:07.807Z 13 260-279 108 108 108 173 173 173 F F F
123042024-02-08T06:44:07.807Z 14 280-299 110 110 110 176 176 176 T F T <---
123052024-02-08T06:44:07.807Z 15 300-319 109 109 109 175 175 175 T F T <---
123062024-02-08T06:44:07.807Z 16 320-339 106 106 106 171 171 171 F F F
123072024-02-08T06:44:07.807Z 17 340-359 103 103 103 167 167 167 T F T <---
123082024-02-08T06:44:07.807Z 18 360-379 104 104 104 169 169 169 F F F
123092024-02-08T06:44:07.807Z 19 380-399 112 110 112 184 178 184 F F F <---
123102024-02-08T06:44:07.807Z 20 400-419 106 106 106 171 171 171 T F T <---
123112024-02-08T06:44:07.807Z 21 420-439 110 110 110 178 178 178 T F T <---
123122024-02-08T06:44:07.807Z 22 440-459 112 108 112 182 174 182 F F F <---
123132024-02-08T06:44:07.807Z 23 460-479 112 101 112 184 162 184 T F T <---
123142024-02-08T06:44:07.808Z 24 480-499 111 111 111 181 181 181 F F F
123152024-02-08T06:44:07.808Z 25 500-519 111 111 111 181 181 181 F F F
123162024-02-08T06:44:07.808Z 26 520-539 112 111 112 184 181 184 F F F <---
123172024-02-08T06:44:07.808Z 27 540-559 112 110 112 184 176 184 T F T <---
123182024-02-08T06:44:07.808Z 28 560-579 112 110 112 182 176 182 T F T <---
123192024-02-08T06:44:07.808Z 29 580-599 108 108 108 173 173 173 F F F
123202024-02-08T06:44:07.808ZMax gen: 112, Max flush: 184
123212024-02-08T06:44:07.808ZError: Difference in extent metadata found!
123222024-02-08T06:44:07.808ZA Difference in extent metadata is expected here
123232024-02-08T06:44:07.808ZOn loop 56
123242024-02-08T06:44:07.808Z
123252024-02-08T06:44:07.812ZVerifying data now
123262024-02-08T06:44:12.451ZNo change after verify
123272024-02-08T06:44:12.451ZLoop: 56 Downstairs dump after verify (and repair):
123282024-02-08T06:44:12.466ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123292024-02-08T06:44:12.466Z 0 000-019 95 95 95 153 153 153 F F F
123302024-02-08T06:44:12.466Z 1 020-039 111 111 111 181 181 181 F F F
123312024-02-08T06:44:12.466Z 2 040-059 111 111 111 181 181 181 F F F
123322024-02-08T06:44:12.466Z 3 060-079 112 112 112 184 184 184 F F F
123332024-02-08T06:44:12.466Z 4 080-099 112 112 112 184 184 184 F F F
123342024-02-08T06:44:12.466Z 5 100-119 104 104 104 168 168 168 F F F
123352024-02-08T06:44:12.466Z 6 120-139 111 111 111 181 181 181 F F F
123362024-02-08T06:44:12.466Z 7 140-159 104 104 104 169 169 169 F F F
123372024-02-08T06:44:12.467Z 8 160-179 107 107 107 172 172 172 F F F
123382024-02-08T06:44:12.467Z 9 180-199 108 108 108 173 173 173 F F F
123392024-02-08T06:44:12.467Z 10 200-219 108 108 108 174 174 174 F F F
123402024-02-08T06:44:12.470Z 11 220-239 111 111 111 181 181 181 F F F
123412024-02-08T06:44:12.470Z 12 240-259 111 111 111 181 181 181 F F F
123422024-02-08T06:44:12.470Z 13 260-279 108 108 108 173 173 173 F F F
123432024-02-08T06:44:12.470Z 14 280-299 113 113 113 185 185 185 F F F
123442024-02-08T06:44:12.470Z 15 300-319 113 113 113 185 185 185 F F F
123452024-02-08T06:44:12.470Z 16 320-339 106 106 106 171 171 171 F F F
123462024-02-08T06:44:12.470Z 17 340-359 113 113 113 185 185 185 F F F
123472024-02-08T06:44:12.470Z 18 360-379 104 104 104 169 169 169 F F F
123482024-02-08T06:44:12.471Z 19 380-399 112 112 112 184 184 184 F F F
123492024-02-08T06:44:12.471Z 20 400-419 113 113 113 185 185 185 F F F
123502024-02-08T06:44:12.471Z 21 420-439 113 113 113 185 185 185 F F F
123512024-02-08T06:44:12.471Z 22 440-459 112 112 112 182 182 182 F F F
123522024-02-08T06:44:12.471Z 23 460-479 113 113 113 185 185 185 F F F
123532024-02-08T06:44:12.471Z 24 480-499 111 111 111 181 181 181 F F F
123542024-02-08T06:44:12.471Z 25 500-519 111 111 111 181 181 181 F F F
123552024-02-08T06:44:12.471Z 26 520-539 112 112 112 184 184 184 F F F
123562024-02-08T06:44:12.471Z 27 540-559 113 113 113 185 185 185 F F F
123572024-02-08T06:44:12.471Z 28 560-579 113 113 113 185 185 185 F F F
123582024-02-08T06:44:12.471Z 29 580-599 108 108 108 173 173 173 F F F
123592024-02-08T06:44:12.471ZMax gen: 113, Max flush: 185
123602024-02-08T06:44:12.471Z
123612024-02-08T06:44:12.471ZBegin loop 57
123622024-02-08T06:44:12.471ZDownstairs to restart: 2
123632024-02-08T06:44:12.487Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715155089Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594}
123642024-02-08T06:44:12.487Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"Crucible runtime is spawned
123652024-02-08T06:44:12.488Z1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715462536Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594}
123662024-02-08T06:44:12.493Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715511572Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594}
123672024-02-08T06:44:12.493Z{"msg":"Crucible 9131239c-e1a5-4ab4-835a-3d5a8322b21a has session id: 0cc3b5ce-15fe-4659-ab12-eb0a8a047516","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715547769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123682024-02-08T06:44:12.494Z{"msg":"Upstairs opts: Upstairs UUID: 9131239c-e1a5-4ab4-835a-3d5a8322b21a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715592466Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123692024-02-08T06:44:12.494Z{"msg":"Crucible stats registered with UUID: 9131239c-e1a5-4ab4-835a-3d5a8322b21a","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715625734Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123702024-02-08T06:44:12.494Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715641152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594}
123712024-02-08T06:44:12.494Z{"msg":"9131239c-e1a5-4ab4-835a-3d5a8322b21a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715661291Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123722024-02-08T06:44:12.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.71567481Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123732024-02-08T06:44:12.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715715117Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123742024-02-08T06:44:12.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715731786Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123752024-02-08T06:44:12.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715748994Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123762024-02-08T06:44:12.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715789771Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123772024-02-08T06:44:12.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715812629Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123782024-02-08T06:44:12.494Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715841827Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123792024-02-08T06:44:12.494Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715866656Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123802024-02-08T06:44:12.494Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.715892323Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123812024-02-08T06:44:12.494Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.716011185Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123822024-02-08T06:44:12.494Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.716049082Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123832024-02-08T06:44:12.495Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.716084969Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123842024-02-08T06:44:12.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.716819544Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123852024-02-08T06:44:12.495Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.716854931Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123862024-02-08T06:44:12.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.71687077Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123872024-02-08T06:44:12.495Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.716948554Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123882024-02-08T06:44:12.495Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.717361703Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123892024-02-08T06:44:12.495Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.717501863Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123902024-02-08T06:44:12.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.717851516Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123912024-02-08T06:44:12.495Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.717889924Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123922024-02-08T06:44:12.495Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.717916331Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123932024-02-08T06:44:12.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.717947679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123942024-02-08T06:44:12.495Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.717972727Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123952024-02-08T06:44:12.495Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.718000545Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123962024-02-08T06:44:12.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.718132315Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123972024-02-08T06:44:12.495Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.718175522Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123982024-02-08T06:44:12.495Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.718645687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
123992024-02-08T06:44:12.496Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719079954Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124002024-02-08T06:44:12.496Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719117172Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124012024-02-08T06:44:12.496Z{"msg":"[0]R flush_numbers[0..12]: [153, 181, 181, 184, 184, 168, 181, 169, 172, 173, 174, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.71913636Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124022024-02-08T06:44:12.496Z{"msg":"[0]R generation[0..12]: [95, 111, 111, 112, 112, 104, 111, 104, 107, 108, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719169228Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124032024-02-08T06:44:12.496Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719200875Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124042024-02-08T06:44:12.496Z{"msg":"[1]R flush_numbers[0..12]: [153, 181, 181, 184, 184, 168, 181, 169, 172, 173, 174, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719235323Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124052024-02-08T06:44:12.496Z{"msg":"[1]R generation[0..12]: [95, 111, 111, 112, 112, 104, 111, 104, 107, 108, 108, 111]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
124062024-02-08T06:44:12.496Z,"time":"2024-02-08T06:44:11.71926721Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124072024-02-08T06:44:12.496Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719288239Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124082024-02-08T06:44:12.496Z{"msg":"[2]R flush_numbers[0..12]: [153, 181, 181, 184, 184, 168, 181, 169, 172, 173, 174, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719318416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124092024-02-08T06:44:12.501Z{"msg":"[2]R generation[0..12]: [95, 111, 111, 112, 112, 104, 111, 104, 107, 108, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719354663Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124102024-02-08T06:44:12.501Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719381652Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124112024-02-08T06:44:12.501Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719412159Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124122024-02-08T06:44:12.501Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719434797Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124132024-02-08T06:44:12.501Z{"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719456446Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124142024-02-08T06:44:12.501Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719479364Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124152024-02-08T06:44:12.501Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719503003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124162024-02-08T06:44:12.502Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.71953258Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124172024-02-08T06:44:12.502Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719556468Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124182024-02-08T06:44:12.502Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719582886Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124192024-02-08T06:44:12.502Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719606205Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124202024-02-08T06:44:12.502Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719631773Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124212024-02-08T06:44:12.502Z{"msg":"9131239c-e1a5-4ab4-835a-3d5a8322b21a is now active with session: 0cc3b5ce-15fe-4659-ab12-eb0a8a047516","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719655111Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124222024-02-08T06:44:12.502Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719686359Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594,"session_id":"0cc3b5ce-15fe-4659-ab12-eb0a8a047516"}
124232024-02-08T06:44:12.502Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:11.719709627Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594}
124242024-02-08T06:44:12.502ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
124252024-02-08T06:44:12.502ZLoading write count information from file "/tmp/test_repair_verify.data"
124262024-02-08T06:44:12.502ZRun Repair workload
124272024-02-08T06:44:12.502Z01/30 Write block 559 len 36864 data: 8 10 10 10 9 8 9 9 9
124282024-02-08T06:44:12.502Z02/30 Write block 181 len 36864 data: 10 11 7 7 10 10 10 10 9
124292024-02-08T06:44:12.502Z03/30 Read block 464 len 24576
124302024-02-08T06:44:12.502Z04/30 Write block 137 len 8192 data: 9 9
124312024-02-08T06:44:12.502Z05/30 Read block 546 len 28672
124322024-02-08T06:44:12.503Z06/30 Read block 514 len 8192
124332024-02-08T06:44:12.503Z07/30 Flush
124342024-02-08T06:44:12.506Z08/30 Write block 82 len 28672 data: 8 8 8 8 8 9 9
124352024-02-08T06:44:12.506Z09/30 Flush
124362024-02-08T06:44:12.506Z10/30 Flush
124372024-02-08T06:44:12.506Z11/30 Write block 241 len 24576 data: 9 8 12 9 9 9
124382024-02-08T06:44:12.506Z12/30 Write block 290 len 32768 data: 11 12 10 8 6 5 6 5
124392024-02-08T06:44:12.506Z13/30 Read block 214 len 8192
124402024-02-08T06:44:12.506Z14/30 Read block 97 len 12288
124412024-02-08T06:44:12.510Z15/30 Write block 19 len 12288 data: 8 8 7
124422024-02-08T06:44:12.510Z16/30 Read block 112 len 8192
124432024-02-08T06:44:12.510Z17/30 Write block 154 len 12288 data: 11 10 11
124442024-02-08T06:44:12.510Z18/30 Read block 265 len 32768
124452024-02-08T06:44:12.510Z19/30 Read block 417 len 4096
124462024-02-08T06:44:12.513Z20/30 Read block 326 len 36864
124472024-02-08T06:44:12.513Z21/30 Write block 384 len 20480 data: 11 9 9 8 9
124482024-02-08T06:44:12.513Z22/30 Flush
124492024-02-08T06:44:12.517Z23/30 Write block 31 len 12288 data: 10 9 9
124502024-02-08T06:44:12.517Z24/30 Read block 403 len 8192
124512024-02-08T06:44:12.517Z25/30 Read block 468 len 20480
124522024-02-08T06:44:12.517Z26/30 Flush
124532024-02-08T06:44:12.520Z27/30 Write block 196 len 28672 data: 9 12 13 14 13 11 12
124542024-02-08T06:44:12.520Z28/30 Read block 593 len 16384
124552024-02-08T06:44:12.520Z29/30 Read block 0 len 8192
124562024-02-08T06:44:12.527Z30/30 Write block 455 len 16384 data: 12 11 11 12
124572024-02-08T06:44:12.527Z----------------------------------------------------------------
124582024-02-08T06:44:12.527Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
124592024-02-08T06:44:12.527ZStates: Active Active Active
124602024-02-08T06:44:12.527ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
124612024-02-08T06:44:12.527Z 1 Acked 1000 Write 9 Done Done Sent false
124622024-02-08T06:44:12.527Z 2 Acked 1001 Write 9 Done Done Sent false
124632024-02-08T06:44:12.527Z 3 Acked 1002 Read 6 Done Done Sent false
124642024-02-08T06:44:12.527Z 4 Acked 1003 Write 2 Done Done Sent false
124652024-02-08T06:44:12.527Z 5 Acked 1004 Read 7 Done Done Sent false
124662024-02-08T06:44:12.527Z 6 Acked 1005 Read 2 Done Done Sent false
124672024-02-08T06:44:12.527Z 7 Acked 1006 Flush 0 Done Done Sent false
124682024-02-08T06:44:12.527Z 8 Acked 1007 Write 7 Done Done Sent false
124692024-02-08T06:44:12.527Z 9 Acked 1008 Flush 0 Done Done Sent false
124702024-02-08T06:44:12.528Z 10 Acked 1009 Flush 0 Done Done Sent false
124712024-02-08T06:44:12.528Z 11 Acked 1010 Write 6 Done Done Sent false
124722024-02-08T06:44:12.528Z 12 Acked 1011 Write 8 Done Done Sent false
124732024-02-08T06:44:12.528Z 13 Acked 1012 Read 2 Done Done Sent false
124742024-02-08T06:44:12.528Z 14 Acked 1013 Read 3 Done Done Sent false
124752024-02-08T06:44:12.528Z 15 Acked 1014 Write 3 Done Done Sent false
124762024-02-08T06:44:12.528Z 16 Acked 1015 Read 2 Done Done Sent false
124772024-02-08T06:44:12.528Z 17 Acked 1016 Write 3 Done Done Sent false
124782024-02-08T06:44:12.528Z 18 Acked 1017 Read 8 Done Done Sent false
124792024-02-08T06:44:12.528Z 19 Acked 1018 Read 1 Done Done Sent false
124802024-02-08T06:44:12.528Z 20 Acked 1019 Read 9 Done Done Sent false
124812024-02-08T06:44:12.528Z 21 Acked 1020 Write 5 Done Done Sent false
124822024-02-08T06:44:12.528Z 22 Acked 1021 Flush 0 Done Done Sent false
124832024-02-08T06:44:12.528Z 23 Acked 1022 Write 3 Done Done Sent false
124842024-02-08T06:44:12.528Z 24 Acked 1023 Read 2 Done Done Sent false
124852024-02-08T06:44:12.529Z 25 Acked 1024 Read 5 Done Done Sent false
124862024-02-08T06:44:12.529Z 26 Acked 1025 Flush 0 Done Done Sent false
124872024-02-08T06:44:12.529Z 27 Acked 1026 Write 7 Done Done Sent false
124882024-02-08T06:44:12.529Z 28 Acked 1027 Read 4 Done Done Sent false
124892024-02-08T06:44:12.529Z 29 Acked 1028 Read 2 Done Done Sent false
124902024-02-08T06:44:12.529Z 30 Acked 1029 Write 4 Sent Sent Sent false
124912024-02-08T06:44:12.529Z STATES DS:0 DS:1 DS:2 TOTAL
124922024-02-08T06:44:12.529Z New 0 0 0 0
124932024-02-08T06:44:12.529Z Sent 1 1 30 32
124942024-02-08T06:44:12.529Z Done 29 29 0 58
124952024-02-08T06:44:12.529Z Skipped 0 0 0 0
124962024-02-08T06:44:12.529Z Error 0 0 0 0
124972024-02-08T06:44:12.529ZLast Flush: 1025 1025 0
124982024-02-08T06:44:12.529ZDownstairs last five completed:
124992024-02-08T06:44:12.529ZUpstairs last five completed: 30 29 28 27 26
125002024-02-08T06:44:12.529Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:11.749762601Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1594}
125012024-02-08T06:44:12.530ZWrote out file "/tmp/test_repair_verify.data"
125022024-02-08T06:44:12.534Z
125032024-02-08T06:44:12.534ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
125042024-02-08T06:44:12.545ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125052024-02-08T06:44:12.545Z 0 000-019 114 114 95 189 189 153 F F F <---
125062024-02-08T06:44:12.545Z 1 020-039 114 114 111 190 190 181 F F F <---
125072024-02-08T06:44:12.545Z 2 040-059 111 111 111 181 181 181 F F F
125082024-02-08T06:44:12.545Z 3 060-079 112 112 112 184 184 184 F F F
125092024-02-08T06:44:12.545Z 4 080-099 114 114 112 187 187 184 F F F <---
125102024-02-08T06:44:12.545Z 5 100-119 104 104 104 168 168 168 F F F
125112024-02-08T06:44:12.545Z 6 120-139 114 114 111 186 186 181 F F F <---
125122024-02-08T06:44:12.545Z 7 140-159 114 114 104 189 189 169 F F F <---
125132024-02-08T06:44:12.548Z 8 160-179 107 107 107 172 172 172 F F F
125142024-02-08T06:44:12.549Z 9 180-199 114 114 108 186 186 173 T T F <---
125152024-02-08T06:44:12.549Z 10 200-219 108 108 108 174 174 174 T T F <---
125162024-02-08T06:44:12.549Z 11 220-239 111 111 111 181 181 181 F F F
125172024-02-08T06:44:12.549Z 12 240-259 114 114 111 189 189 181 F F F <---
125182024-02-08T06:44:12.549Z 13 260-279 108 108 108 173 173 173 F F F
125192024-02-08T06:44:12.549Z 14 280-299 114 114 113 189 189 185 F F F <---
125202024-02-08T06:44:12.549Z 15 300-319 113 113 113 185 185 185 F F F
125212024-02-08T06:44:12.549Z 16 320-339 106 106 106 171 171 171 F F F
125222024-02-08T06:44:12.549Z 17 340-359 113 113 113 185 185 185 F F F
125232024-02-08T06:44:12.549Z 18 360-379 104 104 104 169 169 169 F F F
125242024-02-08T06:44:12.549Z 19 380-399 114 114 112 189 189 184 F F F <---
125252024-02-08T06:44:12.549Z 20 400-419 113 113 113 185 185 185 F F F
125262024-02-08T06:44:12.549Z 21 420-439 113 113 113 185 185 185 F F F
125272024-02-08T06:44:12.550Z 22 440-459 112 112 112 182 182 182 T T F <---
125282024-02-08T06:44:12.550Z 23 460-479 113 113 113 185 185 185 F F F
125292024-02-08T06:44:12.550Z 24 480-499 111 111 111 181 181 181 F F F
125302024-02-08T06:44:12.550Z 25 500-519 111 111 111 181 181 181 F F F
125312024-02-08T06:44:12.550Z 26 520-539 112 112 112 184 184 184 F F F
125322024-02-08T06:44:12.550Z 27 540-559 114 114 113 186 186 185 F F F <---
125332024-02-08T06:44:12.550Z 28 560-579 114 114 113 186 186 185 F F F <---
125342024-02-08T06:44:12.550Z 29 580-599 108 108 108 173 173 173 F F F
125352024-02-08T06:44:12.550ZMax gen: 114, Max flush: 190
125362024-02-08T06:44:12.550ZError: Difference in extent metadata found!
125372024-02-08T06:44:12.550ZA Difference in extent metadata is expected here
125382024-02-08T06:44:12.550ZOn loop 57
125392024-02-08T06:44:12.550Z
125402024-02-08T06:44:12.553ZVerifying data now
125412024-02-08T06:44:17.195ZNo change after verify
125422024-02-08T06:44:17.195ZLoop: 57 Downstairs dump after verify (and repair):
125432024-02-08T06:44:17.199ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125442024-02-08T06:44:17.199Z 0 000-019 114 114 114 189 189 189 F F F
125452024-02-08T06:44:17.199Z 1 020-039 114 114 114 190 190 190 F F F
125462024-02-08T06:44:17.199Z 2 040-059 111 111 111 181 181 181 F F F
125472024-02-08T06:44:17.199Z 3 060-079 112 112 112 184 184 184 F F F
125482024-02-08T06:44:17.199Z 4 080-099 114 114 114 187 187 187 F F F
125492024-02-08T06:44:17.199Z 5 100-119 104 104 104 168 168 168 F F F
125502024-02-08T06:44:17.199Z 6 120-139 114 114 114 186 186 186 F F F
125512024-02-08T06:44:17.199Z 7 140-159 114 114 114 189 189 189 F F F
125522024-02-08T06:44:17.199Z 8 160-179 107 107 107 172 172 172 F F F
125532024-02-08T06:44:17.199Z 9 180-199 115 115 115 191 191 191 F F F
125542024-02-08T06:44:17.199Z 10 200-219 115 115 115 191 191 191 F F F
125552024-02-08T06:44:17.199Z 11 220-239 111 111 111 181 181 181 F F F
125562024-02-08T06:44:17.200Z 12 240-259 114 114 114 189 189 189 F F F
125572024-02-08T06:44:17.200Z 13 260-279 108 108 108 173 173 173 F F F
125582024-02-08T06:44:17.200Z 14 280-299 114 114 114 189 189 189 F F F
125592024-02-08T06:44:17.200Z 15 300-319 113 113 113 185 185 185 F F F
125602024-02-08T06:44:17.200Z 16 320-339 106 106 106 171 171 171 F F F
125612024-02-08T06:44:17.200Z 17 340-359 113 113 113 185 185 185 F F F
125622024-02-08T06:44:17.200Z 18 360-379 104 104 104 169 169 169 F F F
125632024-02-08T06:44:17.200Z 19 380-399 114 114 114 189 189 189 F F F
125642024-02-08T06:44:17.200Z 20 400-419 113 113 113 185 185 185 F F F
125652024-02-08T06:44:17.200Z 21 420-439 113 113 113 185 185 185 F F F
125662024-02-08T06:44:17.200Z 22 440-459 115 115 115 191 191 191 F F F
125672024-02-08T06:44:17.200Z 23 460-479 113 113 113 185 185 185 F F F
125682024-02-08T06:44:17.200Z 24 480-499 111 111 111 181 181 181 F F F
125692024-02-08T06:44:17.200Z 25 500-519 111 111 111 181 181 181 F F F
125702024-02-08T06:44:17.200Z 26 520-539 112 112 112 184 184 184 F F F
125712024-02-08T06:44:17.201Z 27 540-559 114 114 114 186 186 186 F F F
125722024-02-08T06:44:17.201Z 28 560-579 114 114 114 186 186 186 F F F
125732024-02-08T06:44:17.201Z 29 580-599 108 108 108 173 173 173 F F F
125742024-02-08T06:44:17.201ZMax gen: 115, Max flush: 191
125752024-02-08T06:44:17.201Z
125762024-02-08T06:44:17.201ZBegin loop 58
125772024-02-08T06:44:17.201ZDownstairs to restart: 0
125782024-02-08T06:44:17.214Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.441665062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602}
125792024-02-08T06:44:17.214Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.441920853Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602}
125802024-02-08T06:44:17.214Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.441946091Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602}
125812024-02-08T06:44:17.214Z{"msg":"Crucible d54e1293-b776-4186-bfc9-ca9e6348c648 has session id: a65196ea-f1c3-42d8-bc76-0a88bfa34442","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.441979089Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"Crucible runtime is spawned
125822024-02-08T06:44:17.219Z}
125832024-02-08T06:44:17.219Z{"msg":"Upstairs opts: Upstairs UUID: d54e1293-b776-4186-bfc9-ca9e6348c648, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442032675Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125842024-02-08T06:44:17.220Z{"msg":"Crucible stats registered with UUID: d54e1293-b776-4186-bfc9-ca9e6348c648","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442074832Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125852024-02-08T06:44:17.220Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.44209071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602}
125862024-02-08T06:44:17.220Z{"msg":"d54e1293-b776-4186-bfc9-ca9e6348c648 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442174454Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125872024-02-08T06:44:17.220Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442201262Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125882024-02-08T06:44:17.220Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442221281Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125892024-02-08T06:44:17.220Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442254688Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125902024-02-08T06:44:17.220Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442290066Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125912024-02-08T06:44:17.220Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442315673Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125922024-02-08T06:44:17.220Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442356101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125932024-02-08T06:44:17.220Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442385298Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125942024-02-08T06:44:17.220Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442411796Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"io task","client":"2","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125952024-02-08T06:44:17.220Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442436975Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"io task","client":"1","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125962024-02-08T06:44:17.220Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442463493Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"io task","client":"2","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125972024-02-08T06:44:17.220Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:16.44248992Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125982024-02-08T06:44:17.220Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:16.442518368Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
125992024-02-08T06:44:17.221Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442594783Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"io task","client":"1","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126002024-02-08T06:44:17.221Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:16.44263205Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126012024-02-08T06:44:17.221Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442651888Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126022024-02-08T06:44:17.221Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442691265Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126032024-02-08T06:44:17.221Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.442791898Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126042024-02-08T06:44:17.221Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.44302685Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126052024-02-08T06:44:17.221Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.443083506Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126062024-02-08T06:44:17.221Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.443115504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126072024-02-08T06:44:17.221Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.443150311Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126082024-02-08T06:44:17.221Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.443588998Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126092024-02-08T06:44:17.221Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.443717288Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126102024-02-08T06:44:17.221Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.444164345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126112024-02-08T06:44:17.221Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.444201152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126122024-02-08T06:44:17.221Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.444216511Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126132024-02-08T06:44:17.221Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.444236249Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126142024-02-08T06:44:17.221Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.444249958Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126152024-02-08T06:44:17.222Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:16.444270387Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126162024-02-08T06:44:18.217Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.443797305Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126172024-02-08T06:44:18.224Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.444232782Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126182024-02-08T06:44:18.224Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.444783891Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126192024-02-08T06:44:18.224Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.444840806Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126202024-02-08T06:44:18.224Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.445288303Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126212024-02-08T06:44:18.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.445775516Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126222024-02-08T06:44:18.224Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.445808274Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126232024-02-08T06:44:18.225Z{"msg":"[0]R flush_numbers[0..12]: [189, 190, 181, 184, 187, 168, 186, 189, 172, 191, 191, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.445825432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126242024-02-08T06:44:18.225Z{"msg":"[0]R generation[0..12]: [114, 114, 111, 112, 114, 104, 114, 114, 107, 115, 115, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.445847321Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126252024-02-08T06:44:18.225Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.44586259Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126262024-02-08T06:44:18.225Z{"msg":"[1]R flush_numbers[0..12]: [189, 190, 181, 184, 187, 168, 186, 189, 172, 191, 191, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.445884028Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126272024-02-08T06:44:18.225Z{"msg":"[1]R generation[0..12]: [114, 114, 111, 112, 114, 104, 114, 114, 107, 115, 115, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.445898797Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126282024-02-08T06:44:18.225Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.445919905Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126292024-02-08T06:44:18.225Z{"msg":"[2]R flush_numbers[0..12]: [189, 190, 181, 184, 187, 168, 186, 189, 172, 191, 191, 181]","Wait for a query_work_queue command to finish before sending IO
126302024-02-08T06:44:18.225Zv":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.445981841Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126312024-02-08T06:44:18.225Z{"msg":"[2]R generation[0..12]: [114, 114, 111, 112, 114, 104, 114, 114, 107, 115, 115, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446009009Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126322024-02-08T06:44:18.226Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446024837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126332024-02-08T06:44:18.226Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446052965Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126342024-02-08T06:44:18.226Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446067824Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126352024-02-08T06:44:18.226Z{"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446082543Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126362024-02-08T06:44:18.226Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446109101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126372024-02-08T06:44:18.226Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.44612403Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126382024-02-08T06:44:18.226Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446136299Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126392024-02-08T06:44:18.226Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446157638Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126402024-02-08T06:44:18.227Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446171117Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126412024-02-08T06:44:18.227Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446185256Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126422024-02-08T06:44:18.227Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446204204Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126432024-02-08T06:44:18.227Z{"msg":"d54e1293-b776-4186-bfc9-ca9e6348c648 is now active with session: a65196ea-f1c3-42d8-bc76-0a88bfa34442","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446217373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126442024-02-08T06:44:18.227Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446236581Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602,"session_id":"a65196ea-f1c3-42d8-bc76-0a88bfa34442"}
126452024-02-08T06:44:18.227Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:17.446249171Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602}
126462024-02-08T06:44:18.227ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
126472024-02-08T06:44:18.227ZLoading write count information from file "/tmp/test_repair_verify.data"
126482024-02-08T06:44:18.227ZRun Repair workload
126492024-02-08T06:44:18.228Z01/30 Write block 138 len 24576 data: 10 9 7 7 6 6
126502024-02-08T06:44:18.228Z02/30 Flush
126512024-02-08T06:44:18.232Z03/30 Write block 22 len 28672 data: 8 8 13 14 13 13 13
126522024-02-08T06:44:18.232Z04/30 Write block 91 len 28672 data: 13 15 14 15 15 15 15
126532024-02-08T06:44:18.232Z05/30 Read block 147 len 28672
126542024-02-08T06:44:18.233Z06/30 Write block 334 len 36864 data: 13 12 14 12 13 13 13 13 12
126552024-02-08T06:44:18.233Z07/30 Read block 572 len 40960
126562024-02-08T06:44:18.233Z08/30 Read block 226 len 40960
126572024-02-08T06:44:18.233Z09/30 Read block 477 len 8192
126582024-02-08T06:44:18.233Z10/30 Read block 107 len 28672
126592024-02-08T06:44:18.233Z11/30 Read block 418 len 4096
126602024-02-08T06:44:18.233Z12/30 Read block 292 len 4096
126612024-02-08T06:44:18.237Z13/30 Read block 429 len 20480
126622024-02-08T06:44:18.237Z14/30 Read block 443 len 32768
126632024-02-08T06:44:18.237Z15/30 Read block 456 len 24576
126642024-02-08T06:44:18.237Z16/30 Read block 485 len 4096
126652024-02-08T06:44:18.237Z17/30 Read block 175 len 16384
126662024-02-08T06:44:18.237Z18/30 Read block 66 len 8192
126672024-02-08T06:44:18.241Z19/30 Read block 238 len 32768
126682024-02-08T06:44:18.241Z20/30 Write block 199 len 8192 data: 15 14
126692024-02-08T06:44:18.241Z21/30 Write block 262 len 32768 data: 13 11 11 12 11 12 12 9
126702024-02-08T06:44:18.241Z22/30 Write block 64 len 36864 data: 5 5 4 5 9 9 5 5 5
126712024-02-08T06:44:18.241Z23/30 Read block 12 len 4096
126722024-02-08T06:44:18.241Z24/30 Write block 233 len 4096 data: 9
126732024-02-08T06:44:18.245Z25/30 Read block 111 len 40960
126742024-02-08T06:44:18.245Z26/30 Read block 354 len 24576
126752024-02-08T06:44:18.245Z27/30 Read block 590 len 24576
126762024-02-08T06:44:18.245Z28/30 Read block 530 len 28672
126772024-02-08T06:44:18.249Z29/30 Write block 304 len 40960 data: 12 13 11 8 7 7 7 6 7 8
126782024-02-08T06:44:18.250Z30/30 Write block 332 len 32768 data: 9 11 14 13 15 13 14 14
126792024-02-08T06:44:18.250Z----------------------------------------------------------------
126802024-02-08T06:44:18.250Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:28
126812024-02-08T06:44:18.250ZStates: Active Active Active
126822024-02-08T06:44:18.250ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
126832024-02-08T06:44:18.250Z 3 Acked 1002 Write 7 Done Done Done false
126842024-02-08T06:44:18.250Z 4 Acked 1003 Write 7 Done Done Done false
126852024-02-08T06:44:18.250Z 5 Acked 1004 Read 7 Sent Done Done false
126862024-02-08T06:44:18.250Z 6 Acked 1005 Write 9 Sent Done Done false
126872024-02-08T06:44:18.250Z 7 Acked 1006 Read 10 Sent Done Done false
126882024-02-08T06:44:18.251Z 8 Acked 1007 Read 10 Sent Done Done false
126892024-02-08T06:44:18.251Z 9 Acked 1008 Read 2 Sent Done Done false
126902024-02-08T06:44:18.251Z 10 Acked 1009 Read 7 Sent Done Done false
126912024-02-08T06:44:18.251Z 11 Acked 1010 Read 1 Sent Done Done false
126922024-02-08T06:44:18.251Z 12 Acked 1011 Read 1 Sent Done Done false
126932024-02-08T06:44:18.251Z 13 Acked 1012 Read 5 Sent Done Done false
126942024-02-08T06:44:18.251Z 14 Acked 1013 Read 8 Sent Done Done false
126952024-02-08T06:44:18.251Z 15 Acked 1014 Read 6 Sent Done Done false
126962024-02-08T06:44:18.251Z 16 Acked 1015 Read 1 Sent Done Done false
126972024-02-08T06:44:18.251Z 17 Acked 1016 Read 4 Sent Done Done false
126982024-02-08T06:44:18.251Z 18 Acked 1017 Read 2 Sent Done Done false
126992024-02-08T06:44:18.251Z 19 Acked 1018 Read 8 Sent Done Done false
127002024-02-08T06:44:18.251Z 20 Acked 1019 Write 2 Sent Done Done false
127012024-02-08T06:44:18.252Z 21 Acked 1020 Write 8 Sent Done Done false
127022024-02-08T06:44:18.252Z 22 Acked 1021 Write 9 Sent Done Done false
127032024-02-08T06:44:18.252Z 23 Acked 1022 Read 1 Sent Done Done false
127042024-02-08T06:44:18.252Z 24 Acked 1023 Write 1 Sent Done Done false
127052024-02-08T06:44:18.252Z 25 Acked 1024 Read 10 Sent Done Done false
127062024-02-08T06:44:18.252Z 26 Acked 1025 Read 6 Sent Done Done false
127072024-02-08T06:44:18.252Z 27 Acked 1026 Read 6 Sent Done Done false
127082024-02-08T06:44:18.252Z 28 Acked 1027 Read 7 Sent Done Done false
127092024-02-08T06:44:18.252Z 29 Acked 1028 Write 10 Sent Sent Sent false
127102024-02-08T06:44:18.252Z 30 Acked 1029 Write 8 Sent Sent Sent false
127112024-02-08T06:44:18.252Z STATES DS:0 DS:1 DS:2 TOTAL
127122024-02-08T06:44:18.252Z New 0 0 0 0
127132024-02-08T06:44:18.252Z Sent 26 2 2 30
127142024-02-08T06:44:18.253Z Done 2 26 26 54
127152024-02-08T06:44:18.253Z Skipped 0 0 0 0
127162024-02-08T06:44:18.253Z Error 0 0 0 0
127172024-02-08T06:44:18.253ZLast Flush: 1001 1001 1001
127182024-02-08T06:44:18.253ZDownstairs last five completed: 1001 1000
127192024-02-08T06:44:18.253ZUpstairs last five completed: 30 29 28 27 26
127202024-02-08T06:44:18.253Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:17.47449935Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1602}
127212024-02-08T06:44:18.253ZWrote out file "/tmp/test_repair_verify.data"
127222024-02-08T06:44:18.256Z
127232024-02-08T06:44:18.257ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
127242024-02-08T06:44:18.286ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127252024-02-08T06:44:18.286Z 0 000-019 114 114 114 189 189 189 F F F
127262024-02-08T06:44:18.286Z 1 020-039 114 114 114 190 190 190 T T T
127272024-02-08T06:44:18.286Z 2 040-059 111 111 111 181 181 181 F F F
127282024-02-08T06:44:18.286Z 3 060-079 112 112 112 184 184 184 F T T <---
127292024-02-08T06:44:18.286Z 4 080-099 114 114 114 187 187 187 T T T
127302024-02-08T06:44:18.286Z 5 100-119 104 104 104 168 168 168 F F F
127312024-02-08T06:44:18.287Z 6 120-139 116 116 116 192 192 192 F F F
127322024-02-08T06:44:18.287Z 7 140-159 116 116 116 192 192 192 F F F
127332024-02-08T06:44:18.287Z 8 160-179 107 107 107 172 172 172 F F F
127342024-02-08T06:44:18.287Z 9 180-199 115 115 115 191 191 191 F T T <---
127352024-02-08T06:44:18.287Z 10 200-219 115 115 115 191 191 191 F T T <---
127362024-02-08T06:44:18.291Z 11 220-239 111 111 111 181 181 181 F T T <---
127372024-02-08T06:44:18.291Z 12 240-259 114 114 114 189 189 189 F F F
127382024-02-08T06:44:18.291Z 13 260-279 108 108 108 173 173 173 F T T <---
127392024-02-08T06:44:18.291Z 14 280-299 114 114 114 189 189 189 F F F
127402024-02-08T06:44:18.291Z 15 300-319 113 113 113 185 185 185 F T T <---
127412024-02-08T06:44:18.291Z 16 320-339 106 106 106 171 171 171 F T T <---
127422024-02-08T06:44:18.291Z 17 340-359 113 113 113 185 185 185 F T T <---
127432024-02-08T06:44:18.291Z 18 360-379 104 104 104 169 169 169 F F F
127442024-02-08T06:44:18.291Z 19 380-399 114 114 114 189 189 189 F F F
127452024-02-08T06:44:18.292Z 20 400-419 113 113 113 185 185 185 F F F
127462024-02-08T06:44:18.292Z 21 420-439 113 113 113 185 185 185 F F F
127472024-02-08T06:44:18.292Z 22 440-459 115 115 115 191 191 191 F F F
127482024-02-08T06:44:18.292Z 23 460-479 113 113 113 185 185 185 F F F
127492024-02-08T06:44:18.292Z 24 480-499 111 111 111 181 181 181 F F F
127502024-02-08T06:44:18.292Z 25 500-519 111 111 111 181 181 181 F F F
127512024-02-08T06:44:18.292Z 26 520-539 112 112 112 184 184 184 F F F
127522024-02-08T06:44:18.292Z 27 540-559 114 114 114 186 186 186 F F F
127532024-02-08T06:44:18.292Z 28 560-579 114 114 114 186 186 186 F F F
127542024-02-08T06:44:18.292Z 29 580-599 108 108 108 173 173 173 F F F
127552024-02-08T06:44:18.292ZMax gen: 116, Max flush: 192
127562024-02-08T06:44:18.292ZError: Difference in extent metadata found!
127572024-02-08T06:44:18.292ZA Difference in extent metadata is expected here
127582024-02-08T06:44:18.293ZOn loop 58
127592024-02-08T06:44:18.293Z
127602024-02-08T06:44:18.293ZVerifying data now
127612024-02-08T06:44:19.807ZNo change after verify
127622024-02-08T06:44:19.807ZLoop: 58 Downstairs dump after verify (and repair):
127632024-02-08T06:44:19.821ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127642024-02-08T06:44:19.821Z 0 000-019 114 114 114 189 189 189 F F F
127652024-02-08T06:44:19.821Z 1 020-039 117 117 117 193 193 193 F F F
127662024-02-08T06:44:19.825Z 2 040-059 111 111 111 181 181 181 F F F
127672024-02-08T06:44:19.825Z 3 060-079 117 117 117 193 193 193 F F F
127682024-02-08T06:44:19.825Z 4 080-099 117 117 117 193 193 193 F F F
127692024-02-08T06:44:19.825Z 5 100-119 104 104 104 168 168 168 F F F
127702024-02-08T06:44:19.825Z 6 120-139 116 116 116 192 192 192 F F F
127712024-02-08T06:44:19.825Z 7 140-159 116 116 116 192 192 192 F F F
127722024-02-08T06:44:19.825Z 8 160-179 107 107 107 172 172 172 F F F
127732024-02-08T06:44:19.825Z 9 180-199 117 117 117 193 193 193 F F F
127742024-02-08T06:44:19.826Z 10 200-219 117 117 117 193 193 193 F F F
127752024-02-08T06:44:19.826Z 11 220-239 117 117 117 193 193 193 F F F
127762024-02-08T06:44:19.826Z 12 240-259 114 114 114 189 189 189 F F F
127772024-02-08T06:44:19.826Z 13 260-279 117 117 117 193 193 193 F F F
127782024-02-08T06:44:19.826Z 14 280-299 114 114 114 189 189 189 F F F
127792024-02-08T06:44:19.826Z 15 300-319 117 117 117 193 193 193 F F F
127802024-02-08T06:44:19.826Z 16 320-339 117 117 117 193 193 193 F F F
127812024-02-08T06:44:19.826Z 17 340-359 117 117 117 193 193 193 F F F
127822024-02-08T06:44:19.826Z 18 360-379 104 104 104 169 169 169 F F F
127832024-02-08T06:44:19.826Z 19 380-399 114 114 114 189 189 189 F F F
127842024-02-08T06:44:19.826Z 20 400-419 113 113 113 185 185 185 F F F
127852024-02-08T06:44:19.826Z 21 420-439 113 113 113 185 185 185 F F F
127862024-02-08T06:44:19.826Z 22 440-459 115 115 115 191 191 191 F F F
127872024-02-08T06:44:19.826Z 23 460-479 113 113 113 185 185 185 F F F
127882024-02-08T06:44:19.826Z 24 480-499 111 111 111 181 181 181 F F F
127892024-02-08T06:44:19.826Z 25 500-519 111 111 111 181 181 181 F F F
127902024-02-08T06:44:19.827Z 26 520-539 112 112 112 184 184 184 F F F
127912024-02-08T06:44:19.827Z 27 540-559 114 114 114 186 186 186 F F F
127922024-02-08T06:44:19.827Z 28 560-579 114 114 114 186 186 186 F F F
127932024-02-08T06:44:19.827Z 29 580-599 108 108 108 173 173 173 F F F
127942024-02-08T06:44:19.827ZMax gen: 117, Max flush: 193
127952024-02-08T06:44:19.827Z
127962024-02-08T06:44:19.827ZBegin loop 59
127972024-02-08T06:44:19.827ZDownstairs to restart: 0
127982024-02-08T06:44:19.842Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.068858068Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610}
127992024-02-08T06:44:19.842ZCrucible runtime is spawned
128002024-02-08T06:44:19.847Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069216551Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610}
128012024-02-08T06:44:19.847Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069255338Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610}
128022024-02-08T06:44:19.847Z{"msg":"Crucible 078b83a1-cbd3-410e-a91c-dc6600e3830e has session id: 3623147e-c0d6-4c9c-a1ad-2cee47404fdf","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069280346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128032024-02-08T06:44:19.847Z{"msg":"Upstairs opts: Upstairs UUID: 078b83a1-cbd3-410e-a91c-dc6600e3830e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069309534Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128042024-02-08T06:44:19.848Z{"msg":"Crucible stats registered with UUID: 078b83a1-cbd3-410e-a91c-dc6600e3830e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069338152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128052024-02-08T06:44:19.848Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069352151Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610}
128062024-02-08T06:44:19.848Z{"msg":"078b83a1-cbd3-410e-a91c-dc6600e3830e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069381849Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128072024-02-08T06:44:19.848Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069402437Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128082024-02-08T06:44:19.848Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069446074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128092024-02-08T06:44:19.848Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069469042Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128102024-02-08T06:44:19.848Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.06949079Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128112024-02-08T06:44:19.848Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069525818Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128122024-02-08T06:44:19.848Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069541726Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128132024-02-08T06:44:19.848Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069556485Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"io task","client":"2","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128142024-02-08T06:44:19.848Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069579354Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128152024-02-08T06:44:19.848Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069602922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"io task","client":"0","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128162024-02-08T06:44:19.848Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069698015Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128172024-02-08T06:44:19.848Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069743531Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"io task","client":"2","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128182024-02-08T06:44:19.849Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.069773449Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"io task","client":"0","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128192024-02-08T06:44:19.849Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.070414781Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128202024-02-08T06:44:19.849Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.070454168Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128212024-02-08T06:44:19.849Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.070547701Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128222024-02-08T06:44:19.849Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.070593978Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128232024-02-08T06:44:19.849Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.071045864Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128242024-02-08T06:44:19.849Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.071152396Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128252024-02-08T06:44:19.849Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.071419896Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128262024-02-08T06:44:19.849Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.071464092Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128272024-02-08T06:44:19.849Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.071601092Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128282024-02-08T06:44:19.849Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.071661577Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128292024-02-08T06:44:19.849Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.071726382Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128302024-02-08T06:44:19.849Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.07176009Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128312024-02-08T06:44:19.849Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.071781958Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128322024-02-08T06:44:19.849Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.071797287Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128332024-02-08T06:44:19.850Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072038049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128342024-02-08T06:44:19.850Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072475696Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128352024-02-08T06:44:19.850Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072516073Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128362024-02-08T06:44:19.850Z{"msg":"[0]R flush_numbers[0..12]: [189, 193, 181, 193, 193, 168, 192, 192, 172, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072550291Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128372024-02-08T06:44:19.855Z{"msg":"[0]R generation[0..12]: [114, 117, 111, 117, 117, 104, 116, 116, 107, 117, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072586378Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128382024-02-08T06:44:19.855Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072625105Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128392024-02-08T06:44:19.855ZWait for a query_work_queue command to finish before sending IO
128402024-02-08T06:44:19.855Z{"msg":"[1]R flush_numbers[0..12]: [189, 193, 181, 193, 193, 168, 192, 192, 172, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072660602Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128412024-02-08T06:44:19.855Z{"msg":"[1]R generation[0..12]: [114, 117, 111, 117, 117, 104, 116, 116, 107, 117, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.07269515Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128422024-02-08T06:44:19.855Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072734027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128432024-02-08T06:44:19.855Z{"msg":"[2]R flush_numbers[0..12]: [189, 193, 181, 193, 193, 168, 192, 192, 172, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072762455Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128442024-02-08T06:44:19.855Z{"msg":"[2]R generation[0..12]: [114, 117, 111, 117, 117, 104, 116, 116, 107, 117, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072778074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128452024-02-08T06:44:19.855Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072800382Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128462024-02-08T06:44:19.855Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.0728203Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128472024-02-08T06:44:19.856Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072844758Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128482024-02-08T06:44:19.856Z{"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072858777Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128492024-02-08T06:44:19.856Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072879026Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128502024-02-08T06:44:19.856Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072893575Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128512024-02-08T06:44:19.856Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072912503Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128522024-02-08T06:44:19.856Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072926092Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128532024-02-08T06:44:19.856Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072945641Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128542024-02-08T06:44:19.856Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.07295924Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128552024-02-08T06:44:19.856Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072972889Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128562024-02-08T06:44:19.856Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
128572024-02-08T06:44:19.856Z078b83a1-cbd3-410e-a91c-dc6600e3830e is now active with session: 3623147e-c0d6-4c9c-a1ad-2cee47404fdf","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.072988778Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128582024-02-08T06:44:19.856Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.073007226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610,"session_id":"3623147e-c0d6-4c9c-a1ad-2cee47404fdf"}
128592024-02-08T06:44:19.856Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:19.073019675Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610}
128602024-02-08T06:44:19.856ZLoading write count information from file "/tmp/test_repair_verify.data"
128612024-02-08T06:44:19.856ZRun Repair workload
128622024-02-08T06:44:19.856Z01/30 Write block 72 len 4096 data: 6
128632024-02-08T06:44:19.857Z02/30 Read block 381 len 16384
128642024-02-08T06:44:19.857Z03/30 Read block 268 len 12288
128652024-02-08T06:44:19.857Z04/30 Flush
128662024-02-08T06:44:19.857Z05/30 Write block 543 len 8192 data: 8 8
128672024-02-08T06:44:19.857Z06/30 Write block 332 len 32768 data: 10 12 15 14 16 14 15 15
128682024-02-08T06:44:19.857Z07/30 Read block 318 len 4096
128692024-02-08T06:44:19.857Z08/30 Write block 9 len 16384 data: 7 9 7 7
128702024-02-08T06:44:19.857Z09/30 Write block 407 len 12288 data: 8 7 8
128712024-02-08T06:44:19.860Z10/30 Read block 274 len 28672
128722024-02-08T06:44:19.860Z11/30 Read block 236 len 8192
128732024-02-08T06:44:19.860Z12/30 Flush
128742024-02-08T06:44:19.860Z13/30 Write block 20 len 8192 data: 9 8
128752024-02-08T06:44:19.861Z14/30 Read block 83 len 32768
128762024-02-08T06:44:19.861Z15/30 Flush
128772024-02-08T06:44:19.864Z16/30 Write block 320 len 32768 data: 14 12 13 13 11 12 10 7
128782024-02-08T06:44:19.864Z17/30 Write block 559 len 28672 data: 9 11 11 11 10 9 10
128792024-02-08T06:44:19.864Z18/30 Read block 425 len 16384
128802024-02-08T06:44:19.867Z19/30 Read block 330 len 20480
128812024-02-08T06:44:19.867Z20/30 Read block 160 len 36864
128822024-02-08T06:44:19.867Z21/30 Read block 104 len 12288
128832024-02-08T06:44:19.867Z22/30 Read block 505 len 32768
128842024-02-08T06:44:19.870Z23/30 Read block 132 len 4096
128852024-02-08T06:44:19.870Z24/30 Write block 426 len 12288 data: 7 5 5
128862024-02-08T06:44:19.870Z25/30 Read block 376 len 24576
128872024-02-08T06:44:19.870Z26/30 Read block 330 len 8192
128882024-02-08T06:44:19.874Z27/30 Write block 203 len 32768 data: 10 10 10 10 11 8 8 8
128892024-02-08T06:44:19.874Z28/30 Write block 31 len 4096 data: 11
128902024-02-08T06:44:19.874Z29/30 Write block 284 len 20480 data: 8 7 6 7 8
128912024-02-08T06:44:19.874Z30/30 Read block 204 len 16384
128922024-02-08T06:44:19.874Z----------------------------------------------------------------
128932024-02-08T06:44:19.874Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
128942024-02-08T06:44:19.875ZStates: Active Active Active
128952024-02-08T06:44:19.875ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
128962024-02-08T06:44:19.875Z 1 Acked 1000 Write 1 Done Done Done false
128972024-02-08T06:44:19.875Z 2 Acked 1001 Read 4 Done Done Done false
128982024-02-08T06:44:19.875Z 3 Acked 1002 Read 3 Done Done Done false
128992024-02-08T06:44:19.875Z 4 Acked 1003 Flush 0 Sent Done Done false
129002024-02-08T06:44:19.875Z 5 Acked 1004 Write 2 Sent Done Done false
129012024-02-08T06:44:19.875Z 6 Acked 1005 Write 8 Sent Done Done false
129022024-02-08T06:44:19.875Z 7 Acked 1006 Read 1 Sent Done Done false
129032024-02-08T06:44:19.875Z 8 Acked 1007 Write 4 Sent Done Done false
129042024-02-08T06:44:19.875Z 9 Acked 1008 Write 3 Sent Done Done false
129052024-02-08T06:44:19.875Z 10 Acked 1009 Read 7 Sent Done Done false
129062024-02-08T06:44:19.875Z 11 Acked 1010 Read 2 Sent Done Done false
129072024-02-08T06:44:19.875Z 12 Acked 1011 Flush 0 Sent Done Done false
129082024-02-08T06:44:19.875Z 13 Acked 1012 Write 2 Sent Done Done false
129092024-02-08T06:44:19.876Z 14 Acked 1013 Read 8 Sent Done Done false
129102024-02-08T06:44:19.876Z 15 Acked 1014 Flush 0 Sent Done Done false
129112024-02-08T06:44:19.876Z 16 Acked 1015 Write 8 Sent Done Done false
129122024-02-08T06:44:19.876Z 17 Acked 1016 Write 7 Sent Done Done false
129132024-02-08T06:44:19.876Z 18 Acked 1017 Read 4 Sent Done Done false
129142024-02-08T06:44:19.876Z 19 Acked 1018 Read 5 Sent Done Done false
129152024-02-08T06:44:19.876Z 20 Acked 1019 Read 9 Sent Done Done false
129162024-02-08T06:44:19.876Z 21 Acked 1020 Read 3 Sent Done Done false
129172024-02-08T06:44:19.876Z 22 Acked 1021 Read 8 Sent Done Done false
129182024-02-08T06:44:19.876Z 23 Acked 1022 Read 1 Sent Done Done false
129192024-02-08T06:44:19.876Z 24 Acked 1023 Write 3 Sent Done Done false
129202024-02-08T06:44:19.876Z 25 Acked 1024 Read 6 Sent Done Done false
129212024-02-08T06:44:19.876Z 26 Acked 1025 Read 2 Sent Done Done false
129222024-02-08T06:44:19.876Z 27 Acked 1026 Write 8 Sent Done Done false
129232024-02-08T06:44:19.876Z 28 Acked 1027 Write 1 Sent Done Done false
129242024-02-08T06:44:19.876Z 29 Acked 1028 Write 5 Sent Done Done false
129252024-02-08T06:44:19.877Z 30 Acked 1029 Read 4 Sent Done Done false
129262024-02-08T06:44:19.877Z STATES DS:0 DS:1 DS:2 TOTAL
129272024-02-08T06:44:19.877Z New 0 0 0 0
129282024-02-08T06:44:19.877Z Sent 27 0 0 27
129292024-02-08T06:44:19.877Z Done 3 30 30 63
129302024-02-08T06:44:19.877Z Skipped 0 0 0 0
129312024-02-08T06:44:19.877Z Error 0 0 0 0
129322024-02-08T06:44:19.877ZLast Flush: 0 1014 1014
129332024-02-08T06:44:19.877ZDownstairs last five completed:
129342024-02-08T06:44:19.877ZUpstairs last five completed: 30 29 28 27 26
129352024-02-08T06:44:19.877Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:19.100881113Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1610}
129362024-02-08T06:44:19.877ZWrote out file "/tmp/test_repair_verify.data"
129372024-02-08T06:44:19.884Z
129382024-02-08T06:44:19.884ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
129392024-02-08T06:44:19.897ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129402024-02-08T06:44:19.897Z 0 000-019 114 118 118 189 195 195 F F F <---
129412024-02-08T06:44:19.897Z 1 020-039 117 118 118 193 196 196 F T T <---
129422024-02-08T06:44:19.897Z 2 040-059 111 111 111 181 181 181 F F F
129432024-02-08T06:44:19.897Z 3 060-079 117 118 118 193 194 194 T F F <---
129442024-02-08T06:44:19.898Z 4 080-099 117 117 117 193 193 193 F F F
129452024-02-08T06:44:19.898Z 5 100-119 104 104 104 168 168 168 F F F
129462024-02-08T06:44:19.898Z 6 120-139 116 116 116 192 192 192 F F F
129472024-02-08T06:44:19.898Z 7 140-159 116 116 116 192 192 192 F F F
129482024-02-08T06:44:19.898Z 8 160-179 107 107 107 172 172 172 F F F
129492024-02-08T06:44:19.901Z 9 180-199 117 117 117 193 193 193 F F F
129502024-02-08T06:44:19.901Z 10 200-219 117 117 117 193 193 193 F T T <---
129512024-02-08T06:44:19.901Z 11 220-239 117 117 117 193 193 193 F F F
129522024-02-08T06:44:19.901Z 12 240-259 114 114 114 189 189 189 F F F
129532024-02-08T06:44:19.901Z 13 260-279 117 117 117 193 193 193 F F F
129542024-02-08T06:44:19.901Z 14 280-299 114 114 114 189 189 189 F T T <---
129552024-02-08T06:44:19.901Z 15 300-319 117 117 117 193 193 193 F F F
129562024-02-08T06:44:19.902Z 16 320-339 117 118 118 193 195 195 F T T <---
129572024-02-08T06:44:19.902Z 17 340-359 117 117 117 193 193 193 F F F
129582024-02-08T06:44:19.902Z 18 360-379 104 104 104 169 169 169 F F F
129592024-02-08T06:44:19.902Z 19 380-399 114 114 114 189 189 189 F F F
129602024-02-08T06:44:19.902Z 20 400-419 113 118 118 185 195 195 F F F <---
129612024-02-08T06:44:19.902Z 21 420-439 113 113 113 185 185 185 F T T <---
129622024-02-08T06:44:19.902Z 22 440-459 115 115 115 191 191 191 F F F
129632024-02-08T06:44:19.902Z 23 460-479 113 113 113 185 185 185 F F F
129642024-02-08T06:44:19.902Z 24 480-499 111 111 111 181 181 181 F F F
129652024-02-08T06:44:19.902Z 25 500-519 111 111 111 181 181 181 F F F
129662024-02-08T06:44:19.902Z 26 520-539 112 112 112 184 184 184 F F F
129672024-02-08T06:44:19.902Z 27 540-559 114 118 118 186 195 195 F T T <---
129682024-02-08T06:44:19.902Z 28 560-579 114 114 114 186 186 186 F T T <---
129692024-02-08T06:44:19.902Z 29 580-599 108 108 108 173 173 173 F F F
129702024-02-08T06:44:19.902ZMax gen: 118, Max flush: 196
129712024-02-08T06:44:19.903ZError: Difference in extent metadata found!
129722024-02-08T06:44:19.903ZA Difference in extent metadata is expected here
129732024-02-08T06:44:19.903ZOn loop 59
129742024-02-08T06:44:19.903Z
129752024-02-08T06:44:19.906ZVerifying data now
129762024-02-08T06:44:21.435ZNo change after verify
129772024-02-08T06:44:21.435ZLoop: 59 Downstairs dump after verify (and repair):
129782024-02-08T06:44:21.450ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129792024-02-08T06:44:21.450Z 0 000-019 118 118 118 195 195 195 F F F
129802024-02-08T06:44:21.450Z 1 020-039 119 119 119 197 197 197 F F F
129812024-02-08T06:44:21.451Z 2 040-059 111 111 111 181 181 181 F F F
129822024-02-08T06:44:21.451Z 3 060-079 118 118 118 194 194 194 F F F
129832024-02-08T06:44:21.451Z 4 080-099 117 117 117 193 193 193 F F F
129842024-02-08T06:44:21.451Z 5 100-119 104 104 104 168 168 168 F F F
129852024-02-08T06:44:21.451Z 6 120-139 116 116 116 192 192 192 F F F
129862024-02-08T06:44:21.451Z 7 140-159 116 116 116 192 192 192 F F F
129872024-02-08T06:44:21.454Z 8 160-179 107 107 107 172 172 172 F F F
129882024-02-08T06:44:21.454Z 9 180-199 117 117 117 193 193 193 F F F
129892024-02-08T06:44:21.455Z 10 200-219 119 119 119 197 197 197 F F F
129902024-02-08T06:44:21.455Z 11 220-239 117 117 117 193 193 193 F F F
129912024-02-08T06:44:21.455Z 12 240-259 114 114 114 189 189 189 F F F
129922024-02-08T06:44:21.455Z 13 260-279 117 117 117 193 193 193 F F F
129932024-02-08T06:44:21.455Z 14 280-299 119 119 119 197 197 197 F F F
129942024-02-08T06:44:21.455Z 15 300-319 117 117 117 193 193 193 F F F
129952024-02-08T06:44:21.455Z 16 320-339 119 119 119 197 197 197 F F F
129962024-02-08T06:44:21.455Z 17 340-359 117 117 117 193 193 193 F F F
129972024-02-08T06:44:21.455Z 18 360-379 104 104 104 169 169 169 F F F
129982024-02-08T06:44:21.455Z 19 380-399 114 114 114 189 189 189 F F F
129992024-02-08T06:44:21.455Z 20 400-419 118 118 118 195 195 195 F F F
130002024-02-08T06:44:21.455Z 21 420-439 119 119 119 197 197 197 F F F
130012024-02-08T06:44:21.455Z 22 440-459 115 115 115 191 191 191 F F F
130022024-02-08T06:44:21.455Z 23 460-479 113 113 113 185 185 185 F F F
130032024-02-08T06:44:21.455Z 24 480-499 111 111 111 181 181 181 F F F
130042024-02-08T06:44:21.456Z 25 500-519 111 111 111 181 181 181 F F F
130052024-02-08T06:44:21.456Z 26 520-539 112 112 112 184 184 184 F F F
130062024-02-08T06:44:21.456Z 27 540-559 119 119 119 197 197 197 F F F
130072024-02-08T06:44:21.456Z 28 560-579 119 119 119 197 197 197 F F F
130082024-02-08T06:44:21.456Z 29 580-599 108 108 108 173 173 173 F F F
130092024-02-08T06:44:21.456ZMax gen: 119, Max flush: 197
130102024-02-08T06:44:21.456Z
130112024-02-08T06:44:21.456ZBegin loop 60
130122024-02-08T06:44:21.456ZDownstairs to restart: 0
130132024-02-08T06:44:21.471Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698053941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618}
130142024-02-08T06:44:21.476Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698342879Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618}
130152024-02-08T06:44:21.476Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
130162024-02-08T06:44:21.477Z,"time":"2024-02-08T06:44:20.698380686Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618}
130172024-02-08T06:44:21.477Z{"msg":"Crucible 5ab49e52-b909-4e2b-8ca0-c35ba3017f24 has session id: 364d7aad-eb05-4411-a6f2-316a97496b2b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698413304Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130182024-02-08T06:44:21.477Z{"msg":"Upstairs opts: Upstairs UUID: 5ab49e52-b909-4e2b-8ca0-c35ba3017f24, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.69846491Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130192024-02-08T06:44:21.477Z{"msg":"Crucible stats registered with UUID: 5ab49e52-b909-4e2b-8ca0-c35ba3017f24","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698516616Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130202024-02-08T06:44:21.477Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698536035Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618}
130212024-02-08T06:44:21.477Z{"msg":"5ab49e52-b909-4e2b-8ca0-c35ba3017f24 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698571422Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130222024-02-08T06:44:21.477Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.69859824Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130232024-02-08T06:44:21.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698653486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130242024-02-08T06:44:21.477Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698683643Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130252024-02-08T06:44:21.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698712381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130262024-02-08T06:44:21.477Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698760907Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130272024-02-08T06:44:21.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698801755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130282024-02-08T06:44:21.477Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698846351Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"io task","client":"0","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130292024-02-08T06:44:21.477Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698889348Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130302024-02-08T06:44:21.477Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.698932955Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130312024-02-08T06:44:21.478Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.699005489Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"io task","client":"0","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130322024-02-08T06:44:21.478Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.699050026Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130332024-02-08T06:44:21.478Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.699080144Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130342024-02-08T06:44:21.478Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.699643041Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130352024-02-08T06:44:21.478Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.699715036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130362024-02-08T06:44:21.478Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.699767572Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130372024-02-08T06:44:21.478Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.69980146Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130382024-02-08T06:44:21.478Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.700280013Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130392024-02-08T06:44:21.478Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.700392765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130402024-02-08T06:44:21.478Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.700525135Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130412024-02-08T06:44:21.478Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.70059994Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130422024-02-08T06:44:21.478Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.70073399Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130432024-02-08T06:44:21.478Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.700776056Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130442024-02-08T06:44:21.478Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.700795945Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130452024-02-08T06:44:21.478Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.700939154Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130462024-02-08T06:44:21.479Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.70099119Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130472024-02-08T06:44:21.479Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.701035277Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130482024-02-08T06:44:21.479Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.701227222Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130492024-02-08T06:44:21.484Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.701671919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130502024-02-08T06:44:21.484Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.701716696Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130512024-02-08T06:44:21.484Z{"msg":"[0]R flush_numbers[0..12]: [195, 197, 181, 194, 193, 168, 192, 192, 172, 193, 197, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.701752653Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130522024-02-08T06:44:21.484Z{"msg":"[0]R generation[0..12]: [118, 119, 111, 118, 117, 104, 116, 116, 107, 117, 119, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.701779411Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130532024-02-08T06:44:21.484Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.701815018Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130542024-02-08T06:44:21.484Z{"msg":"[1]R flush_numbers[0..12]: [195, 197, 181, 194, 193, 168, 192, 192, 172, 193, 197, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.701850116Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130552024-02-08T06:44:21.484Z{"msg":"[1]R generation[0..12]: [118, 119, 111, 118, 117, 104, 116, 116, 107, 117, 119, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.701887143Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130562024-02-08T06:44:21.484Z{"Wait for a query_work_queue command to finish before sending IO
130572024-02-08T06:44:21.484Zmsg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.70193066Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130582024-02-08T06:44:21.484Z{"msg":"[2]R flush_numbers[0..12]: [195, 197, 181, 194, 193, 168, 192, 192, 172, 193, 197, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.701972177Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130592024-02-08T06:44:21.484Z{"msg":"[2]R generation[0..12]: [118, 119, 111, 118, 117, 104, 116, 116, 107, 117, 119, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702012914Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130602024-02-08T06:44:21.484Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702051631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130612024-02-08T06:44:21.484Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702094867Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130622024-02-08T06:44:21.484Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702135714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130632024-02-08T06:44:21.485Z{"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702173061Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130642024-02-08T06:44:21.485Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702209959Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130652024-02-08T06:44:21.485Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702246146Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130662024-02-08T06:44:21.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702280023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130672024-02-08T06:44:21.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
130682024-02-08T06:44:21.485Z,"time":"2024-02-08T06:44:20.702315051Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130692024-02-08T06:44:21.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702357148Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130702024-02-08T06:44:21.485Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702392315Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130712024-02-08T06:44:21.485Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702410544Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130722024-02-08T06:44:21.485Z{"msg":"5ab49e52-b909-4e2b-8ca0-c35ba3017f24 is now active with session: 364d7aad-eb05-4411-a6f2-316a97496b2b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702437091Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130732024-02-08T06:44:21.485Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.70245454Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618,"session_id":"364d7aad-eb05-4411-a6f2-316a97496b2b"}
130742024-02-08T06:44:21.485Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:20.702483418Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618}
130752024-02-08T06:44:21.485ZLoading write count information from file "/tmp/test_repair_verify.data"
130762024-02-08T06:44:21.485ZRun Repair workload
130772024-02-08T06:44:21.485Z01/30 Write block 22 len 40960 data: 9 9 14 15 14 14 14 11 11 12
130782024-02-08T06:44:21.486Z02/30 Write block 451 len 40960 data: 7 8 9 9 13 12 12 13 12 8
130792024-02-08T06:44:21.486Z03/30 Write block 208 len 40960 data: 9 9 9 12 14 13 12 10 10 11
130802024-02-08T06:44:21.486Z04/30 Read block 285 len 40960
130812024-02-08T06:44:21.486Z05/30 Read block 111 len 4096
130822024-02-08T06:44:21.486Z06/30 Write block 474 len 8192 data: 10 10
130832024-02-08T06:44:21.486Z07/30 Flush
130842024-02-08T06:44:21.489Z08/30 Write block 305 len 12288 data: 14 12 9
130852024-02-08T06:44:21.489Z09/30 Read block 584 len 36864
130862024-02-08T06:44:21.489Z10/30 Read block 414 len 32768
130872024-02-08T06:44:21.489Z11/30 Read block 579 len 28672
130882024-02-08T06:44:21.490Z12/30 Write block 65 len 20480 data: 6 5 6 10 10
130892024-02-08T06:44:21.490Z13/30 Read block 111 len 24576
130902024-02-08T06:44:21.493Z14/30 Read block 90 len 36864
130912024-02-08T06:44:21.493Z15/30 Write block 266 len 20480 data: 12 13 13 10 7
130922024-02-08T06:44:21.493Z16/30 Read block 548 len 4096
130932024-02-08T06:44:21.493Z17/30 Read block 181 len 12288
130942024-02-08T06:44:21.496Z18/30 Write block 20 len 32768 data: 10 9 10 10 15 16 15 15
130952024-02-08T06:44:21.496Z19/30 Read block 444 len 4096
130962024-02-08T06:44:21.496Z20/30 Read block 522 len 32768
130972024-02-08T06:44:21.499Z21/30 Write block 218 len 24576 data: 10 11 11 12 13 13
130982024-02-08T06:44:21.499Z22/30 Write block 406 len 4096 data: 10
130992024-02-08T06:44:21.499Z23/30 Read block 431 len 4096
131002024-02-08T06:44:21.499Z24/30 Read block 372 len 12288
131012024-02-08T06:44:21.499Z25/30 Write block 538 len 8192 data: 9 7
131022024-02-08T06:44:21.503Z26/30 Write block 581 len 12288 data: 9 10 10
131032024-02-08T06:44:21.503Z27/30 Write block 397 len 36864 data: 7 7 8 8 10 10 9 10 10
131042024-02-08T06:44:21.503Z28/30 Read block 183 len 4096
131052024-02-08T06:44:21.503Z29/30 Write block 258 len 8192 data: 7 10
131062024-02-08T06:44:21.503Z30/30 Read block 203 len 16384
131072024-02-08T06:44:21.509Z----------------------------------------------------------------
131082024-02-08T06:44:21.509Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
131092024-02-08T06:44:21.509ZStates: Active Active Active
131102024-02-08T06:44:21.509ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
131112024-02-08T06:44:21.509Z 1 Acked 1000 Write 10 Sent Done Done false
131122024-02-08T06:44:21.510Z 2 Acked 1001 Write 10 Sent Done Done false
131132024-02-08T06:44:21.510Z 3 Acked 1002 Write 10 Sent Done Done false
131142024-02-08T06:44:21.510Z 4 Acked 1003 Read 10 Sent Done Done false
131152024-02-08T06:44:21.510Z 5 Acked 1004 Read 1 Sent Done Done false
131162024-02-08T06:44:21.510Z 6 Acked 1005 Write 2 Sent Done Done false
131172024-02-08T06:44:21.510Z 7 Acked 1006 Flush 0 Sent Done Done false
131182024-02-08T06:44:21.510Z 8 Acked 1007 Write 3 Sent Done Done false
131192024-02-08T06:44:21.510Z 9 Acked 1008 Read 9 Sent Done Done false
131202024-02-08T06:44:21.510Z 10 Acked 1009 Read 8 Sent Done Done false
131212024-02-08T06:44:21.510Z 11 Acked 1010 Read 7 Sent Done Done false
131222024-02-08T06:44:21.510Z 12 Acked 1011 Write 5 Sent Done Done false
131232024-02-08T06:44:21.510Z 13 Acked 1012 Read 6 Sent Done Done false
131242024-02-08T06:44:21.510Z 14 Acked 1013 Read 9 Sent Done Done false
131252024-02-08T06:44:21.510Z 15 Acked 1014 Write 5 Sent Done Done false
131262024-02-08T06:44:21.511Z 16 Acked 1015 Read 1 Sent Done Done false
131272024-02-08T06:44:21.511Z 17 Acked 1016 Read 3 Sent Done Done false
131282024-02-08T06:44:21.511Z 18 Acked 1017 Write 8 Sent Done Done false
131292024-02-08T06:44:21.511Z 19 Acked 1018 Read 1 Sent Done Done false
131302024-02-08T06:44:21.511Z 20 Acked 1019 Read 8 Sent Done Done false
131312024-02-08T06:44:21.511Z 21 Acked 1020 Write 6 Sent Done Done false
131322024-02-08T06:44:21.511Z 22 Acked 1021 Write 1 Sent Done Done false
131332024-02-08T06:44:21.511Z 23 Acked 1022 Read 1 Sent Done Done false
131342024-02-08T06:44:21.511Z 24 Acked 1023 Read 3 Sent Done Done false
131352024-02-08T06:44:21.511Z 25 Acked 1024 Write 2 Sent Done Done false
131362024-02-08T06:44:21.511Z 26 Acked 1025 Write 3 Sent Done Done false
131372024-02-08T06:44:21.511Z 27 Acked 1026 Write 9 Sent Done Done false
131382024-02-08T06:44:21.511Z 28 Acked 1027 Read 1 Sent Done Done false
131392024-02-08T06:44:21.511Z 29 Acked 1028 Write 2 Sent Done Done false
131402024-02-08T06:44:21.512Z 30 Acked 1029 Read 4 Sent Done Done false
131412024-02-08T06:44:21.512Z STATES DS:0 DS:1 DS:2 TOTAL
131422024-02-08T06:44:21.512Z New 0 0 0 0
131432024-02-08T06:44:21.512Z Sent 30 0 0 30
131442024-02-08T06:44:21.512Z Done 0 30 30 60
131452024-02-08T06:44:21.512Z Skipped 0 0 0 0
131462024-02-08T06:44:21.512Z Error 0 0 0 0
131472024-02-08T06:44:21.512ZLast Flush: 0 1006 1006
131482024-02-08T06:44:21.512ZDownstairs last five completed:
131492024-02-08T06:44:21.512ZUpstairs last five completed: 30 29 28 27 26
131502024-02-08T06:44:21.512Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:20.731045774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1618}
131512024-02-08T06:44:21.512ZWrote out file "/tmp/test_repair_verify.data"
131522024-02-08T06:44:21.516Z
131532024-02-08T06:44:21.516ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
131542024-02-08T06:44:21.529ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131552024-02-08T06:44:21.529Z 0 000-019 118 118 118 195 195 195 F F F
131562024-02-08T06:44:21.529Z 1 020-039 119 120 120 197 198 198 F T T <---
131572024-02-08T06:44:21.530Z 2 040-059 111 111 111 181 181 181 F F F
131582024-02-08T06:44:21.530Z 3 060-079 118 118 118 194 194 194 F T T <---
131592024-02-08T06:44:21.530Z 4 080-099 117 117 117 193 193 193 F F F
131602024-02-08T06:44:21.533Z 5 100-119 104 104 104 168 168 168 F F F
131612024-02-08T06:44:21.533Z 6 120-139 116 116 116 192 192 192 F F F
131622024-02-08T06:44:21.533Z 7 140-159 116 116 116 192 192 192 F F F
131632024-02-08T06:44:21.533Z 8 160-179 107 107 107 172 172 172 F F F
131642024-02-08T06:44:21.534Z 9 180-199 117 117 117 193 193 193 F F F
131652024-02-08T06:44:21.534Z 10 200-219 119 120 120 197 198 198 F T T <---
131662024-02-08T06:44:21.534Z 11 220-239 117 117 117 193 193 193 F T T <---
131672024-02-08T06:44:21.534Z 12 240-259 114 114 114 189 189 189 F T T <---
131682024-02-08T06:44:21.534Z 13 260-279 117 117 117 193 193 193 F T T <---
131692024-02-08T06:44:21.534Z 14 280-299 119 119 119 197 197 197 F F F
131702024-02-08T06:44:21.534Z 15 300-319 117 117 117 193 193 193 F T T <---
131712024-02-08T06:44:21.534Z 16 320-339 119 119 119 197 197 197 F F F
131722024-02-08T06:44:21.534Z 17 340-359 117 117 117 193 193 193 F F F
131732024-02-08T06:44:21.534Z 18 360-379 104 104 104 169 169 169 F F F
131742024-02-08T06:44:21.534Z 19 380-399 114 114 114 189 189 189 F T T <---
131752024-02-08T06:44:21.534Z 20 400-419 118 118 118 195 195 195 F T T <---
131762024-02-08T06:44:21.534Z 21 420-439 119 119 119 197 197 197 F F F
131772024-02-08T06:44:21.534Z 22 440-459 115 120 120 191 198 198 F F F <---
131782024-02-08T06:44:21.534Z 23 460-479 113 120 120 185 198 198 F F F <---
131792024-02-08T06:44:21.535Z 24 480-499 111 111 111 181 181 181 F F F
131802024-02-08T06:44:21.535Z 25 500-519 111 111 111 181 181 181 F F F
131812024-02-08T06:44:21.535Z 26 520-539 112 112 112 184 184 184 F T T <---
131822024-02-08T06:44:21.535Z 27 540-559 119 119 119 197 197 197 F F F
131832024-02-08T06:44:21.535Z 28 560-579 119 119 119 197 197 197 F F F
131842024-02-08T06:44:21.535Z 29 580-599 108 108 108 173 173 173 F T T <---
131852024-02-08T06:44:21.535ZMax gen: 120, Max flush: 198
131862024-02-08T06:44:21.535ZError: Difference in extent metadata found!
131872024-02-08T06:44:21.535ZA Difference in extent metadata is expected here
131882024-02-08T06:44:21.535ZOn loop 60
131892024-02-08T06:44:21.535Z
131902024-02-08T06:44:21.538ZVerifying data now
131912024-02-08T06:44:26.190ZNo change after verify
131922024-02-08T06:44:26.193ZLoop: 60 Downstairs dump after verify (and repair):
131932024-02-08T06:44:26.205ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131942024-02-08T06:44:26.205Z 0 000-019 118 118 118 195 195 195 F F F
131952024-02-08T06:44:26.205Z 1 020-039 121 121 121 199 199 199 F F F
131962024-02-08T06:44:26.205Z 2 040-059 111 111 111 181 181 181 F F F
131972024-02-08T06:44:26.206Z 3 060-079 121 121 121 199 199 199 F F F
131982024-02-08T06:44:26.206Z 4 080-099 117 117 117 193 193 193 F F F
131992024-02-08T06:44:26.206Z 5 100-119 104 104 104 168 168 168 F F F
132002024-02-08T06:44:26.206Z 6 120-139 116 116 116 192 192 192 F F F
132012024-02-08T06:44:26.206Z 7 140-159 116 116 116 192 192 192 F F F
132022024-02-08T06:44:26.210Z 8 160-179 107 107 107 172 172 172 F F F
132032024-02-08T06:44:26.210Z 9 180-199 117 117 117 193 193 193 F F F
132042024-02-08T06:44:26.210Z 10 200-219 121 121 121 199 199 199 F F F
132052024-02-08T06:44:26.210Z 11 220-239 121 121 121 199 199 199 F F F
132062024-02-08T06:44:26.210Z 12 240-259 121 121 121 199 199 199 F F F
132072024-02-08T06:44:26.210Z 13 260-279 121 121 121 199 199 199 F F F
132082024-02-08T06:44:26.210Z 14 280-299 119 119 119 197 197 197 F F F
132092024-02-08T06:44:26.210Z 15 300-319 121 121 121 199 199 199 F F F
132102024-02-08T06:44:26.210Z 16 320-339 119 119 119 197 197 197 F F F
132112024-02-08T06:44:26.210Z 17 340-359 117 117 117 193 193 193 F F F
132122024-02-08T06:44:26.210Z 18 360-379 104 104 104 169 169 169 F F F
132132024-02-08T06:44:26.210Z 19 380-399 121 121 121 199 199 199 F F F
132142024-02-08T06:44:26.210Z 20 400-419 121 121 121 199 199 199 F F F
132152024-02-08T06:44:26.210Z 21 420-439 119 119 119 197 197 197 F F F
132162024-02-08T06:44:26.211Z 22 440-459 120 120 120 198 198 198 F F F
132172024-02-08T06:44:26.211Z 23 460-479 120 120 120 198 198 198 F F F
132182024-02-08T06:44:26.211Z 24 480-499 111 111 111 181 181 181 F F F
132192024-02-08T06:44:26.211Z 25 500-519 111 111 111 181 181 181 F F F
132202024-02-08T06:44:26.211Z 26 520-539 121 121 121 199 199 199 F F F
132212024-02-08T06:44:26.211Z 27 540-559 119 119 119 197 197 197 F F F
132222024-02-08T06:44:26.211Z 28 560-579 119 119 119 197 197 197 F F F
132232024-02-08T06:44:26.211Z 29 580-599 121 121 121 199 199 199 F F F
132242024-02-08T06:44:26.211ZMax gen: 121, Max flush: 199
132252024-02-08T06:44:26.211Z
132262024-02-08T06:44:26.211ZBegin loop 61
132272024-02-08T06:44:26.211ZDownstairs to restart: 2
132282024-02-08T06:44:26.226Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.452819003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626}
132292024-02-08T06:44:26.231Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453073344Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626}
132302024-02-08T06:44:26.231Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453110381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626}
132312024-02-08T06:44:26.231ZCrucible runtime is spawned
132322024-02-08T06:44:26.232Z{"msg":"Crucible 32a401d9-ded8-408c-b1ab-5ed873e5c0f0 has session id: dba478e1-aade-41d2-8b7d-67a4be753b34","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453151378Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132332024-02-08T06:44:26.232Z{"msg":"Upstairs opts: Upstairs UUID: 32a401d9-ded8-408c-b1ab-5ed873e5c0f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453192075Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132342024-02-08T06:44:26.232Z{"msg":"Crucible stats registered with UUID: 32a401d9-ded8-408c-b1ab-5ed873e5c0f0","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453227212Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132352024-02-08T06:44:26.232Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453251431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626}
132362024-02-08T06:44:26.232Z{"msg":"32a401d9-ded8-408c-b1ab-5ed873e5c0f0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453355643Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132372024-02-08T06:44:26.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.4533981Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132382024-02-08T06:44:26.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453429907Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132392024-02-08T06:44:26.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453471324Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132402024-02-08T06:44:26.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453512481Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132412024-02-08T06:44:26.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453538619Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132422024-02-08T06:44:26.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453607174Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132432024-02-08T06:44:26.232Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453647171Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132442024-02-08T06:44:26.232Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453675909Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"io task","client":"0","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132452024-02-08T06:44:26.232Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453701807Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132462024-02-08T06:44:26.232Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453727105Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132472024-02-08T06:44:26.233Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453753153Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"io task","client":"0","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132482024-02-08T06:44:26.233Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.453826007Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132492024-02-08T06:44:26.233Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.454413323Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132502024-02-08T06:44:26.233Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.45445625Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132512024-02-08T06:44:26.233Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.454493137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132522024-02-08T06:44:26.233Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.454537344Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132532024-02-08T06:44:26.233Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.455082123Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132542024-02-08T06:44:26.233Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.455143259Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132552024-02-08T06:44:26.233Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.455712036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132562024-02-08T06:44:26.233Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.455749203Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132572024-02-08T06:44:26.233Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.455765762Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132582024-02-08T06:44:26.233Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.45578695Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132592024-02-08T06:44:26.233Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.455812409Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132602024-02-08T06:44:26.233Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.455829787Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132612024-02-08T06:44:26.233Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.455855425Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132622024-02-08T06:44:26.233Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.455869024Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132632024-02-08T06:44:26.238Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456337839Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132642024-02-08T06:44:26.238Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456781406Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132652024-02-08T06:44:26.238Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456821753Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132662024-02-08T06:44:26.238Z{"msg":"[0]R flush_numbers[0..12]: [195, 199, 181, 199, 193, 168, 192, 192, 172, 193, 199, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456840511Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132672024-02-08T06:44:26.239Z{"msg":"[0]R generation[0..12]: [118, 121, 111, 121, 117, 104, 116, 116, 107, 117, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.45685565Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132682024-02-08T06:44:26.239Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456877078Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132692024-02-08T06:44:26.239Z{"msg":"[1]R flush_numbers[0..12]: [195, 199, 181, 199, 193, 168, 192, 192, 172, 193, 199, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456891337Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132702024-02-08T06:44:26.239Z{"msg":"[1]R generation[0..12]: [118, 121, 111, 121, 117, 104, 116, 116, 107, 117, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456905196Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132712024-02-08T06:44:26.239Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456925685Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132722024-02-08T06:44:26.239Z{"msg":"[2]R flush_numbers[0..12]: [195, 199, 181, 199, 193, 168, 192, 192, 172, 193, 199, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456940014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132732024-02-08T06:44:26.239Z{"msg":"[2]R generation[0..12]: [118, 121, 111, 121, 117, 104, 116, 116, 107, 117, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456953953Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132742024-02-08T06:44:26.239Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456968752Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132752024-02-08T06:44:26.239Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.456982151Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132762024-02-08T06:44:26.239Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457018948Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO
132772024-02-08T06:44:26.239Z,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132782024-02-08T06:44:26.239Z{"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457044336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132792024-02-08T06:44:26.239Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457057735Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132802024-02-08T06:44:26.239Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457071094Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132812024-02-08T06:44:26.239Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457090582Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132822024-02-08T06:44:26.240Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457104092Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132832024-02-08T06:44:26.240Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.45711726Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132842024-02-08T06:44:26.240Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457135679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132852024-02-08T06:44:26.240Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457151808Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132862024-02-08T06:44:26.240Z{"msg":"32a401d9-ded8-408c-b1ab-5ed873e5c0f0 is now active with session: dba478e1-aade-41d2-8b7d-67a4be753b34","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457165327Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132872024-02-08T06:44:26.240Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457190635Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626,"session_id":"dba478e1-aade-41d2-8b7d-67a4be753b34"}
132882024-02-08T06:44:26.240Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:25.457203714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626}
132892024-02-08T06:44:26.240ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
132902024-02-08T06:44:26.240ZLoading write count information from file "/tmp/test_repair_verify.data"
132912024-02-08T06:44:26.240ZRun Repair workload
132922024-02-08T06:44:26.240Z01/30 Write block 57 len 32768 data: 11 6 5 6 6 8 7 6
132932024-02-08T06:44:26.240Z02/30 Write block 563 len 36864 data: 11 10 11 10 10 9 8 10 10
132942024-02-08T06:44:26.240Z03/30 Flush
132952024-02-08T06:44:26.244Z04/30 Write block 452 len 24576 data: 9 10 10 14 13 13
132962024-02-08T06:44:26.244Z05/30 Read block 112 len 40960
132972024-02-08T06:44:26.244Z06/30 Read block 545 len 40960
132982024-02-08T06:44:26.244Z07/30 Read block 531 len 4096
132992024-02-08T06:44:26.244Z08/30 Write block 314 len 24576 data: 8 4 6 6 8 10
133002024-02-08T06:44:26.244Z09/30 Read block 559 len 40960
133012024-02-08T06:44:26.244Z10/30 Read block 78 len 28672
133022024-02-08T06:44:26.244Z11/30 Read block 485 len 28672
133032024-02-08T06:44:26.247Z12/30 Read block 79 len 32768
133042024-02-08T06:44:26.247Z13/30 Read block 152 len 28672
133052024-02-08T06:44:26.247Z14/30 Write block 562 len 8192 data: 12 12
133062024-02-08T06:44:26.247Z15/30 Read block 288 len 16384
133072024-02-08T06:44:26.250Z16/30 Write block 546 len 20480 data: 9 13 13 15 13
133082024-02-08T06:44:26.251Z17/30 Write block 545 len 28672 data: 8 10 14 14 16 14 14
133092024-02-08T06:44:26.251Z18/30 Read block 108 len 8192
133102024-02-08T06:44:26.251Z19/30 Read block 127 len 8192
133112024-02-08T06:44:26.254Z20/30 Flush
133122024-02-08T06:44:26.254Z21/30 Write block 359 len 8192 data: 10 8
133132024-02-08T06:44:26.254Z22/30 Read block 509 len 20480
133142024-02-08T06:44:26.257Z23/30 Write block 211 len 12288 data: 13 15 14
133152024-02-08T06:44:26.257Z24/30 Read block 411 len 28672
133162024-02-08T06:44:26.257Z25/30 Write block 407 len 24576 data: 9 8 9 7 6 7
133172024-02-08T06:44:26.257Z26/30 Read block 337 len 20480
133182024-02-08T06:44:26.263Z27/30 Write block 571 len 36864 data: 11 10 10 8 8 8 9 9 11
133192024-02-08T06:44:26.263Z28/30 Read block 298 len 32768
133202024-02-08T06:44:26.267Z29/30 Read block 26 len 36864
133212024-02-08T06:44:26.267Z30/30 Read block 412 len 40960
133222024-02-08T06:44:26.267Z----------------------------------------------------------------
133232024-02-08T06:44:26.267Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
133242024-02-08T06:44:26.267ZStates: Active Active Active
133252024-02-08T06:44:26.267ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
133262024-02-08T06:44:26.267Z 1 Acked 1000 Write 8 Done Done Sent false
133272024-02-08T06:44:26.267Z 2 Acked 1001 Write 9 Done Done Sent false
133282024-02-08T06:44:26.268Z 3 Acked 1002 Flush 0 Done Done Sent false
133292024-02-08T06:44:26.268Z 4 Acked 1003 Write 6 Done Done Sent false
133302024-02-08T06:44:26.268Z 5 Acked 1004 Read 10 Done Done Sent false
133312024-02-08T06:44:26.268Z 6 Acked 1005 Read 10 Done Done Sent false
133322024-02-08T06:44:26.268Z 7 Acked 1006 Read 1 Done Done Sent false
133332024-02-08T06:44:26.268Z 8 Acked 1007 Write 6 Done Done Sent false
133342024-02-08T06:44:26.268Z 9 Acked 1008 Read 10 Done Done Sent false
133352024-02-08T06:44:26.268Z 10 Acked 1009 Read 7 Done Done Sent false
133362024-02-08T06:44:26.268Z 11 Acked 1010 Read 7 Done Done Sent false
133372024-02-08T06:44:26.268Z 12 Acked 1011 Read 8 Done Done Sent false
133382024-02-08T06:44:26.268Z 13 Acked 1012 Read 7 Done Done Sent false
133392024-02-08T06:44:26.268Z 14 Acked 1013 Write 2 Done Done Sent false
133402024-02-08T06:44:26.268Z 15 Acked 1014 Read 4 Done Done Sent false
133412024-02-08T06:44:26.268Z 16 Acked 1015 Write 5 Done Done Sent false
133422024-02-08T06:44:26.268Z 17 Acked 1016 Write 7 Done Done Sent false
133432024-02-08T06:44:26.269Z 18 Acked 1017 Read 2 Done Done Sent false
133442024-02-08T06:44:26.269Z 19 Acked 1018 Read 2 Done Done Sent false
133452024-02-08T06:44:26.269Z 20 Acked 1019 Flush 0 Done Done Sent false
133462024-02-08T06:44:26.269Z 21 Acked 1020 Write 2 Done Done Sent false
133472024-02-08T06:44:26.269Z 22 Acked 1021 Read 5 Done Done Sent false
133482024-02-08T06:44:26.269Z 23 Acked 1022 Write 3 Done Done Sent false
133492024-02-08T06:44:26.269Z 24 Acked 1023 Read 7 Done Done Sent false
133502024-02-08T06:44:26.269Z 25 Acked 1024 Write 6 Done Done Sent false
133512024-02-08T06:44:26.269Z 26 Acked 1025 Read 5 Done Done Sent false
133522024-02-08T06:44:26.269Z 27 Acked 1026 Write 9 Done Done Sent false
133532024-02-08T06:44:26.269Z 28 Acked 1027 Read 8 Done Done Sent false
133542024-02-08T06:44:26.269Z 29 Acked 1028 Read 9 Done Done Sent false
133552024-02-08T06:44:26.269Z 30 Acked 1029 Read 10 Done Done Sent false
133562024-02-08T06:44:26.269Z STATES DS:0 DS:1 DS:2 TOTAL
133572024-02-08T06:44:26.269Z New 0 0 0 0
133582024-02-08T06:44:26.269Z Sent 0 0 30 30
133592024-02-08T06:44:26.270Z Done 30 30 0 60
133602024-02-08T06:44:26.270Z Skipped 0 0 0 0
133612024-02-08T06:44:26.270Z Error 0 0 0 0
133622024-02-08T06:44:26.270ZLast Flush: 1019 1019 0
133632024-02-08T06:44:26.270ZDownstairs last five completed:
133642024-02-08T06:44:26.270ZUpstairs last five completed: 30 29 28 27 26
133652024-02-08T06:44:26.270Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:25.489718613Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1626}
133662024-02-08T06:44:26.270ZWrote out file "/tmp/test_repair_verify.data"
133672024-02-08T06:44:26.270Z
133682024-02-08T06:44:26.273ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
133692024-02-08T06:44:26.286ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133702024-02-08T06:44:26.286Z 0 000-019 118 118 118 195 195 195 F F F
133712024-02-08T06:44:26.286Z 1 020-039 121 121 121 199 199 199 F F F
133722024-02-08T06:44:26.286Z 2 040-059 122 122 111 200 200 181 F F F <---
133732024-02-08T06:44:26.286Z 3 060-079 122 122 121 200 200 199 F F F <---
133742024-02-08T06:44:26.286Z 4 080-099 117 117 117 193 193 193 F F F
133752024-02-08T06:44:26.287Z 5 100-119 104 104 104 168 168 168 F F F
133762024-02-08T06:44:26.287Z 6 120-139 116 116 116 192 192 192 F F F
133772024-02-08T06:44:26.287Z 7 140-159 116 116 116 192 192 192 F F F
133782024-02-08T06:44:26.287Z 8 160-179 107 107 107 172 172 172 F F F
133792024-02-08T06:44:26.287Z 9 180-199 117 117 117 193 193 193 F F F
133802024-02-08T06:44:26.287Z 10 200-219 121 121 121 199 199 199 T T F <---
133812024-02-08T06:44:26.290Z 11 220-239 121 121 121 199 199 199 F F F
133822024-02-08T06:44:26.290Z 12 240-259 121 121 121 199 199 199 F F F
133832024-02-08T06:44:26.290Z 13 260-279 121 121 121 199 199 199 F F F
133842024-02-08T06:44:26.290Z 14 280-299 119 119 119 197 197 197 F F F
133852024-02-08T06:44:26.290Z 15 300-319 122 122 121 201 201 199 F F F <---
133862024-02-08T06:44:26.291Z 16 320-339 119 119 119 197 197 197 F F F
133872024-02-08T06:44:26.291Z 17 340-359 117 117 117 193 193 193 T T F <---
133882024-02-08T06:44:26.291Z 18 360-379 104 104 104 169 169 169 T T F <---
133892024-02-08T06:44:26.291Z 19 380-399 121 121 121 199 199 199 F F F
133902024-02-08T06:44:26.291Z 20 400-419 121 121 121 199 199 199 T T F <---
133912024-02-08T06:44:26.291Z 21 420-439 119 119 119 197 197 197 F F F
133922024-02-08T06:44:26.291Z 22 440-459 122 122 120 201 201 198 F F F <---
133932024-02-08T06:44:26.291Z 23 460-479 120 120 120 198 198 198 F F F
133942024-02-08T06:44:26.291Z 24 480-499 111 111 111 181 181 181 F F F
133952024-02-08T06:44:26.291Z 25 500-519 111 111 111 181 181 181 F F F
133962024-02-08T06:44:26.291Z 26 520-539 121 121 121 199 199 199 F F F
133972024-02-08T06:44:26.291Z 27 540-559 122 122 119 201 201 197 F F F <---
133982024-02-08T06:44:26.291Z 28 560-579 122 122 119 201 201 197 T T F <---
133992024-02-08T06:44:26.291Z 29 580-599 121 121 121 199 199 199 F F F
134002024-02-08T06:44:26.291ZMax gen: 122, Max flush: 201
134012024-02-08T06:44:26.291ZError: Difference in extent metadata found!
134022024-02-08T06:44:26.292ZA Difference in extent metadata is expected here
134032024-02-08T06:44:26.292ZOn loop 61
134042024-02-08T06:44:26.292Z
134052024-02-08T06:44:26.295ZVerifying data now
134062024-02-08T06:44:26.818ZNo change after verify
134072024-02-08T06:44:26.818ZLoop: 61 Downstairs dump after verify (and repair):
134082024-02-08T06:44:26.833ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134092024-02-08T06:44:26.833Z 0 000-019 118 118 118 195 195 195 F F F
134102024-02-08T06:44:26.833Z 1 020-039 121 121 121 199 199 199 F F F
134112024-02-08T06:44:26.833Z 2 040-059 122 122 122 200 200 200 F F F
134122024-02-08T06:44:26.833Z 3 060-079 122 122 122 200 200 200 F F F
134132024-02-08T06:44:26.833Z 4 080-099 117 117 117 193 193 193 F F F
134142024-02-08T06:44:26.833Z 5 100-119 104 104 104 168 168 168 F F F
134152024-02-08T06:44:26.837Z 6 120-139 116 116 116 192 192 192 F F F
134162024-02-08T06:44:26.837Z 7 140-159 116 116 116 192 192 192 F F F
134172024-02-08T06:44:26.837Z 8 160-179 107 107 107 172 172 172 F F F
134182024-02-08T06:44:26.837Z 9 180-199 117 117 117 193 193 193 F F F
134192024-02-08T06:44:26.837Z 10 200-219 123 123 123 202 202 202 F F F
134202024-02-08T06:44:26.837Z 11 220-239 121 121 121 199 199 199 F F F
134212024-02-08T06:44:26.837Z 12 240-259 121 121 121 199 199 199 F F F
134222024-02-08T06:44:26.838Z 13 260-279 121 121 121 199 199 199 F F F
134232024-02-08T06:44:26.838Z 14 280-299 119 119 119 197 197 197 F F F
134242024-02-08T06:44:26.838Z 15 300-319 122 122 122 201 201 201 F F F
134252024-02-08T06:44:26.838Z 16 320-339 119 119 119 197 197 197 F F F
134262024-02-08T06:44:26.838Z 17 340-359 123 123 123 202 202 202 F F F
134272024-02-08T06:44:26.838Z 18 360-379 123 123 123 202 202 202 F F F
134282024-02-08T06:44:26.838Z 19 380-399 121 121 121 199 199 199 F F F
134292024-02-08T06:44:26.838Z 20 400-419 123 123 123 202 202 202 F F F
134302024-02-08T06:44:26.838Z 21 420-439 119 119 119 197 197 197 F F F
134312024-02-08T06:44:26.838Z 22 440-459 122 122 122 201 201 201 F F F
134322024-02-08T06:44:26.838Z 23 460-479 120 120 120 198 198 198 F F F
134332024-02-08T06:44:26.838Z 24 480-499 111 111 111 181 181 181 F F F
134342024-02-08T06:44:26.838Z 25 500-519 111 111 111 181 181 181 F F F
134352024-02-08T06:44:26.838Z 26 520-539 121 121 121 199 199 199 F F F
134362024-02-08T06:44:26.838Z 27 540-559 122 122 122 201 201 201 F F F
134372024-02-08T06:44:26.839Z 28 560-579 123 123 123 202 202 202 F F F
134382024-02-08T06:44:26.839Z 29 580-599 121 121 121 199 199 199 F F F
134392024-02-08T06:44:26.839ZMax gen: 123, Max flush: 202
134402024-02-08T06:44:26.839Z
134412024-02-08T06:44:26.839ZBegin loop 62
134422024-02-08T06:44:26.839ZDownstairs to restart: 0
134432024-02-08T06:44:26.854Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.080967103Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634}
134442024-02-08T06:44:26.854Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081221324Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634}
134452024-02-08T06:44:26.860Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081254952Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634}
134462024-02-08T06:44:26.860Z{"msg":"Crucible 8516338c-983c-48af-85a1-20efca941a6f has session id: da618664-ef29-4446-ad54-21b9ec4b3990","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.0812793Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134472024-02-08T06:44:26.860Z{"msg":"Crucible runtime is spawned
134482024-02-08T06:44:26.860ZUpstairs opts: Upstairs UUID: 8516338c-983c-48af-85a1-20efca941a6f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081337226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134492024-02-08T06:44:26.860Z{"msg":"Crucible stats registered with UUID: 8516338c-983c-48af-85a1-20efca941a6f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081372373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134502024-02-08T06:44:26.860Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081396371Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634}
134512024-02-08T06:44:26.860Z{"msg":"8516338c-983c-48af-85a1-20efca941a6f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081469265Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134522024-02-08T06:44:26.860Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081508523Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134532024-02-08T06:44:26.860Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.08154554Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134542024-02-08T06:44:26.860Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081598016Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134552024-02-08T06:44:26.860Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081628294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134562024-02-08T06:44:26.860Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081653502Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134572024-02-08T06:44:26.860Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.08167843Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134582024-02-08T06:44:26.860Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081703248Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134592024-02-08T06:44:26.861Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081740155Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134602024-02-08T06:44:26.861Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081771603Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134612024-02-08T06:44:26.861Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081801731Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134622024-02-08T06:44:26.861Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081829178Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134632024-02-08T06:44:26.861Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.081864206Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134642024-02-08T06:44:26.861Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.08260826Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134652024-02-08T06:44:26.861Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.082697663Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134662024-02-08T06:44:26.861Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.08274075Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134672024-02-08T06:44:26.861Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.082759439Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134682024-02-08T06:44:26.861Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.083285539Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134692024-02-08T06:44:26.861Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.083325766Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134702024-02-08T06:44:26.861Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.083817169Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134712024-02-08T06:44:26.861Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.083854677Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134722024-02-08T06:44:26.861Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.084000806Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134732024-02-08T06:44:26.861Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.084044662Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134742024-02-08T06:44:26.862Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.084082519Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134752024-02-08T06:44:26.862Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.084112447Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134762024-02-08T06:44:26.862Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.084146484Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134772024-02-08T06:44:26.862Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.084180372Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134782024-02-08T06:44:26.862Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.08447476Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134792024-02-08T06:44:26.866Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.084912987Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134802024-02-08T06:44:26.867Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.084943885Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134812024-02-08T06:44:26.867Z{"msg":"[0]R flush_numbers[0..12]: [195, 199, 200, 200, 193, 168, 192, 192, 172, 193, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.084959794Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134822024-02-08T06:44:26.867Z{"msg":"[0]R generation[0..12]: [118, 121, 122, 122, 117, 104, 116, 116, 107, 117, 123, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.084996751Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134832024-02-08T06:44:26.867Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.08501181Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134842024-02-08T06:44:26.867Z{"msg":"[1]R flush_numbers[0..12]: [195, 199, 200, 200, 193, 168, 192, 192, 172, 193, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085026238Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134852024-02-08T06:44:26.867Z{"msg":"[1]R generation[0..12]: [118, 121, 122, 122, 117, 104, 116, 116, 107, 117, 123, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085040597Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134862024-02-08T06:44:26.867Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085055676Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134872024-02-08T06:44:26.867Z{"msg":"[2]R flush_numbers[0..12]: [195, 199, 200, 200, 193, 168, 192, 192, 172, 193, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085070175Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134882024-02-08T06:44:26.867Z{"msg":"[2]R generation[0..12]: [118, 121, 122, 122, 117, 104, 116, 116, 107, 117, 123, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085093244Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134892024-02-08T06:44:26.867Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085107832Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134902024-02-08T06:44:26.867ZWait for a query_work_queue command to finish before sending IO
134912024-02-08T06:44:26.867Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085133541Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134922024-02-08T06:44:26.867Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085147649Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134932024-02-08T06:44:26.867Z{"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085161308Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134942024-02-08T06:44:26.868Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085174817Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134952024-02-08T06:44:26.868Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085195416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134962024-02-08T06:44:26.868Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085207935Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134972024-02-08T06:44:26.868Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085221394Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134982024-02-08T06:44:26.868Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085235413Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
134992024-02-08T06:44:26.868Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085249012Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
135002024-02-08T06:44:26.868Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085262161Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
135012024-02-08T06:44:26.868Z{"msg":"8516338c-983c-48af-85a1-20efca941a6f is now active with session: da618664-ef29-4446-ad54-21b9ec4b3990","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.08527526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
135022024-02-08T06:44:26.868Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085294149Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634,"session_id":"da618664-ef29-4446-ad54-21b9ec4b3990"}
135032024-02-08T06:44:26.868Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:26.085307257Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634}
135042024-02-08T06:44:26.868ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135052024-02-08T06:44:26.868ZLoading write count information from file "/tmp/test_repair_verify.data"
135062024-02-08T06:44:26.868ZRun Repair workload
135072024-02-08T06:44:26.868Z01/30 Write block 133 len 20480 data: 8 7 6 7 10
135082024-02-08T06:44:26.868Z02/30 Flush
135092024-02-08T06:44:26.868Z03/30 Flush
135102024-02-08T06:44:26.869Z04/30 Write block 405 len 40960 data: 11 11 10 9 10 8 7 8 7 8
135112024-02-08T06:44:26.869Z05/30 Flush
135122024-02-08T06:44:26.872Z06/30 Write block 343 len 24576 data: 11 10 11 11 9 7
135132024-02-08T06:44:26.872Z07/30 Write block 558 len 24576 data: 7 10 12 12 13 13
135142024-02-08T06:44:26.872Z08/30 Read block 562 len 32768
135152024-02-08T06:44:26.872Z09/30 Write block 376 len 24576 data: 8 7 7 7 7 5
135162024-02-08T06:44:26.872Z10/30 Write block 554 len 12288 data: 11 11 15
135172024-02-08T06:44:26.872Z11/30 Write block 69 len 32768 data: 11 6 6 7 7 9 8 9
135182024-02-08T06:44:26.872Z12/30 Write block 181 len 24576 data: 11 12 8 8 11 11
135192024-02-08T06:44:26.875Z13/30 Flush
135202024-02-08T06:44:26.876Z14/30 Write block 445 len 40960 data: 9 7 8 9 10 8 8 10 11 11
135212024-02-08T06:44:26.881Z15/30 Write block 433 len 28672 data: 5 8 8 10 12 10 7
135222024-02-08T06:44:26.882Z16/30 Flush
135232024-02-08T06:44:26.885Z17/30 Write block 413 len 28672 data: 8 9 8 7 9 10 10
135242024-02-08T06:44:26.885Z18/30 Write block 250 len 28672 data: 13 13 10 8 9 11 9
135252024-02-08T06:44:26.885Z19/30 Flush
135262024-02-08T06:44:26.885Z20/30 Write block 593 len 28672 data: 10 9 7 8 7 4 4
135272024-02-08T06:44:26.885Z21/30 Write block 0 len 4096 data: 2
135282024-02-08T06:44:26.885Z22/30 Write block 130 len 32768 data: 7 9 9 9 8 7 8 11
135292024-02-08T06:44:26.888Z23/30 Write block 413 len 40960 data: 9 10 9 8 10 11 11 10 9 9
135302024-02-08T06:44:26.888Z24/30 Read block 460 len 12288
135312024-02-08T06:44:26.888Z25/30 Write block 444 len 32768 data: 9 10 8 9 10 11 9 9
135322024-02-08T06:44:26.891Z26/30 Write block 548 len 28672 data: 15 17 15 15 13 12 12
135332024-02-08T06:44:26.892Z27/30 Write block 292 len 16384 data: 11 9 7 6
135342024-02-08T06:44:26.892Z28/30 Write block 435 len 28672 data: 9 11 13 11 8 9 9
135352024-02-08T06:44:26.892Z29/30 Write block 274 len 32768 data: 7 7 8 10 8 12 10 9
135362024-02-08T06:44:26.892Z30/30 Write block 77 len 24576 data: 6 7 8 9 8 9
135372024-02-08T06:44:26.895Z----------------------------------------------------------------
135382024-02-08T06:44:26.895Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
135392024-02-08T06:44:26.895ZStates: Active Active Active
135402024-02-08T06:44:26.896ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
135412024-02-08T06:44:26.896Z 1 Acked 1000 Write 5 Done Done Done false
135422024-02-08T06:44:26.896Z 2 Acked 1001 Flush 0 Sent Done Done false
135432024-02-08T06:44:26.896Z 3 Acked 1002 Flush 0 Sent Done Done false
135442024-02-08T06:44:26.896Z 4 Acked 1003 Write 10 Sent Done Done false
135452024-02-08T06:44:26.896Z 5 Acked 1004 Flush 0 Sent Done Done false
135462024-02-08T06:44:26.896Z 6 Acked 1005 Write 6 Sent Done Done false
135472024-02-08T06:44:26.896Z 7 Acked 1006 Write 6 Sent Done Done false
135482024-02-08T06:44:26.896Z 8 Acked 1007 Read 8 Sent Done Done false
135492024-02-08T06:44:26.896Z 9 Acked 1008 Write 6 Sent Done Done false
135502024-02-08T06:44:26.896Z 10 Acked 1009 Write 3 Sent Done Done false
135512024-02-08T06:44:26.896Z 11 Acked 1010 Write 8 Sent Done Done false
135522024-02-08T06:44:26.896Z 12 Acked 1011 Write 6 Sent Done Done false
135532024-02-08T06:44:26.896Z 13 Acked 1012 Flush 0 Sent Done Done false
135542024-02-08T06:44:26.896Z 14 Acked 1013 Write 10 Sent Done Done false
135552024-02-08T06:44:26.897Z 15 Acked 1014 Write 7 Sent Done Done false
135562024-02-08T06:44:26.897Z 16 Acked 1015 Flush 0 Sent Done Done false
135572024-02-08T06:44:26.897Z 17 Acked 1016 Write 7 Sent Done Done false
135582024-02-08T06:44:26.897Z 18 Acked 1017 Write 7 Sent Done Done false
135592024-02-08T06:44:26.897Z 19 Acked 1018 Flush 0 Sent Done Done false
135602024-02-08T06:44:26.897Z 20 Acked 1019 Write 7 Sent Done Done false
135612024-02-08T06:44:26.897Z 21 Acked 1020 Write 1 Sent Done Done false
135622024-02-08T06:44:26.897Z 22 Acked 1021 Write 8 Sent Done Done false
135632024-02-08T06:44:26.897Z 23 Acked 1022 Write 10 Sent Done Done false
135642024-02-08T06:44:26.897Z 24 Acked 1023 Read 3 Sent Done Done false
135652024-02-08T06:44:26.897Z 25 Acked 1024 Write 8 Sent Done Done false
135662024-02-08T06:44:26.897Z 26 Acked 1025 Write 7 Sent Done Done false
135672024-02-08T06:44:26.897Z 27 Acked 1026 Write 4 Sent Done Done false
135682024-02-08T06:44:26.897Z 28 Acked 1027 Write 7 Sent Done Done false
135692024-02-08T06:44:26.897Z 29 Acked 1028 Write 8 Sent Sent Done false
135702024-02-08T06:44:26.897Z 30 Acked 1029 Write 6 Sent Sent Sent false
135712024-02-08T06:44:26.898Z STATES DS:0 DS:1 DS:2 TOTAL
135722024-02-08T06:44:26.898Z New 0 0 0 0
135732024-02-08T06:44:26.898Z Sent 29 2 1 32
135742024-02-08T06:44:26.898Z Done 1 28 29 58
135752024-02-08T06:44:26.898Z Skipped 0 0 0 0
135762024-02-08T06:44:26.898Z Error 0 0 0 0
135772024-02-08T06:44:26.898ZLast Flush: 0 1018 1018
135782024-02-08T06:44:26.898ZDownstairs last five completed:
135792024-02-08T06:44:26.898ZUpstairs last five completed: 30 29 28 27 26
135802024-02-08T06:44:26.898Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:26.119204503Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1634}
135812024-02-08T06:44:26.898ZWrote out file "/tmp/test_repair_verify.data"
135822024-02-08T06:44:26.901Z
135832024-02-08T06:44:26.902ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
135842024-02-08T06:44:26.918ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135852024-02-08T06:44:26.918Z 0 000-019 118 118 118 195 195 195 F T T <---
135862024-02-08T06:44:26.918Z 1 020-039 121 121 121 199 199 199 F F F
135872024-02-08T06:44:26.918Z 2 040-059 122 122 122 200 200 200 F F F
135882024-02-08T06:44:26.922Z 3 060-079 122 124 124 200 206 206 F T T <---
135892024-02-08T06:44:26.922Z 4 080-099 117 117 117 193 193 193 F T T <---
135902024-02-08T06:44:26.922Z 5 100-119 104 104 104 168 168 168 F F F
135912024-02-08T06:44:26.922Z 6 120-139 116 124 124 192 203 203 T T T <---
135922024-02-08T06:44:26.922Z 7 140-159 116 116 116 192 192 192 F F F
135932024-02-08T06:44:26.922Z 8 160-179 107 107 107 172 172 172 F F F
135942024-02-08T06:44:26.922Z 9 180-199 117 124 124 193 206 206 F F F <---
135952024-02-08T06:44:26.922Z 10 200-219 123 123 123 202 202 202 F F F
135962024-02-08T06:44:26.923Z 11 220-239 121 121 121 199 199 199 F F F
135972024-02-08T06:44:26.923Z 12 240-259 121 124 124 199 208 208 F F F <---
135982024-02-08T06:44:26.923Z 13 260-279 121 121 121 199 199 199 F T T <---
135992024-02-08T06:44:26.923Z 14 280-299 119 119 119 197 197 197 F T T <---
136002024-02-08T06:44:26.923Z 15 300-319 122 122 122 201 201 201 F F F
136012024-02-08T06:44:26.923Z 16 320-339 119 119 119 197 197 197 F F F
136022024-02-08T06:44:26.923Z 17 340-359 123 124 124 202 206 206 F F F <---
136032024-02-08T06:44:26.923Z 18 360-379 123 124 124 202 206 206 F F F <---
136042024-02-08T06:44:26.923Z 19 380-399 121 124 124 199 206 206 F F F <---
136052024-02-08T06:44:26.923Z 20 400-419 123 124 124 202 208 208 F T T <---
136062024-02-08T06:44:26.923Z 21 420-439 119 124 124 197 207 207 F T T <---
136072024-02-08T06:44:26.923Z 22 440-459 122 124 124 201 207 207 F T T <---
136082024-02-08T06:44:26.923Z 23 460-479 120 120 120 198 198 198 F F F
136092024-02-08T06:44:26.923Z 24 480-499 111 111 111 181 181 181 F F F
136102024-02-08T06:44:26.923Z 25 500-519 111 111 111 181 181 181 F F F
136112024-02-08T06:44:26.923Z 26 520-539 121 121 121 199 199 199 F F F
136122024-02-08T06:44:26.924Z 27 540-559 122 124 124 201 206 206 F T T <---
136132024-02-08T06:44:26.924Z 28 560-579 123 124 124 202 206 206 F F F <---
136142024-02-08T06:44:26.924Z 29 580-599 121 121 121 199 199 199 F T T <---
136152024-02-08T06:44:26.924ZMax gen: 124, Max flush: 208
136162024-02-08T06:44:26.924ZError: Difference in extent metadata found!
136172024-02-08T06:44:26.924ZA Difference in extent metadata is expected here
136182024-02-08T06:44:26.924ZOn loop 62
136192024-02-08T06:44:26.924Z
136202024-02-08T06:44:26.927ZVerifying data now
136212024-02-08T06:44:32.755ZNo change after verify
136222024-02-08T06:44:32.755ZLoop: 62 Downstairs dump after verify (and repair):
136232024-02-08T06:44:32.770ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136242024-02-08T06:44:32.770Z 0 000-019 125 125 125 209 209 209 F F F
136252024-02-08T06:44:32.770Z 1 020-039 121 121 121 199 199 199 F F F
136262024-02-08T06:44:32.770Z 2 040-059 122 122 122 200 200 200 F F F
136272024-02-08T06:44:32.770Z 3 060-079 125 125 125 209 209 209 F F F
136282024-02-08T06:44:32.770Z 4 080-099 125 125 125 209 209 209 F F F
136292024-02-08T06:44:32.774Z 5 100-119 104 104 104 168 168 168 F F F
136302024-02-08T06:44:32.774Z 6 120-139 125 125 125 209 209 209 F F F
136312024-02-08T06:44:32.774Z 7 140-159 116 116 116 192 192 192 F F F
136322024-02-08T06:44:32.774Z 8 160-179 107 107 107 172 172 172 F F F
136332024-02-08T06:44:32.774Z 9 180-199 124 124 124 206 206 206 F F F
136342024-02-08T06:44:32.774Z 10 200-219 123 123 123 202 202 202 F F F
136352024-02-08T06:44:32.774Z 11 220-239 121 121 121 199 199 199 F F F
136362024-02-08T06:44:32.774Z 12 240-259 124 124 124 208 208 208 F F F
136372024-02-08T06:44:32.774Z 13 260-279 125 125 125 209 209 209 F F F
136382024-02-08T06:44:32.774Z 14 280-299 125 125 125 209 209 209 F F F
136392024-02-08T06:44:32.775Z 15 300-319 122 122 122 201 201 201 F F F
136402024-02-08T06:44:32.775Z 16 320-339 119 119 119 197 197 197 F F F
136412024-02-08T06:44:32.775Z 17 340-359 124 124 124 206 206 206 F F F
136422024-02-08T06:44:32.775Z 18 360-379 124 124 124 206 206 206 F F F
136432024-02-08T06:44:32.775Z 19 380-399 124 124 124 206 206 206 F F F
136442024-02-08T06:44:32.775Z 20 400-419 125 125 125 209 209 209 F F F
136452024-02-08T06:44:32.775Z 21 420-439 125 125 125 209 209 209 F F F
136462024-02-08T06:44:32.775Z 22 440-459 125 125 125 209 209 209 F F F
136472024-02-08T06:44:32.775Z 23 460-479 120 120 120 198 198 198 F F F
136482024-02-08T06:44:32.775Z 24 480-499 111 111 111 181 181 181 F F F
136492024-02-08T06:44:32.775Z 25 500-519 111 111 111 181 181 181 F F F
136502024-02-08T06:44:32.775Z 26 520-539 121 121 121 199 199 199 F F F
136512024-02-08T06:44:32.775Z 27 540-559 125 125 125 209 209 209 F F F
136522024-02-08T06:44:32.775Z 28 560-579 124 124 124 206 206 206 F F F
136532024-02-08T06:44:32.775Z 29 580-599 125 125 125 209 209 209 F F F
136542024-02-08T06:44:32.775ZMax gen: 125, Max flush: 209
136552024-02-08T06:44:32.776Z
136562024-02-08T06:44:32.776ZBegin loop 63
136572024-02-08T06:44:32.776ZDownstairs to restart: 1
136582024-02-08T06:44:32.792Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.01763848Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642}
136592024-02-08T06:44:32.792Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.017890511Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642}
136602024-02-08T06:44:32.792Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.01790979Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642}
136612024-02-08T06:44:32.797Z{"msg":"Crucible d12a31ab-12d6-4e65-a489-d93f2ceeb2b7 has session id: 912cb1fe-b51a-4168-bade-907f57902088","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.017942687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136622024-02-08T06:44:32.797Z{Crucible runtime is spawned
136632024-02-08T06:44:32.797Z"msg":"Upstairs opts: Upstairs UUID: d12a31ab-12d6-4e65-a489-d93f2ceeb2b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018073897Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136642024-02-08T06:44:32.797Z{"msg":"Crucible stats registered with UUID: d12a31ab-12d6-4e65-a489-d93f2ceeb2b7","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018127024Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136652024-02-08T06:44:32.798Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018190319Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642}
136662024-02-08T06:44:32.798Z{"msg":"d12a31ab-12d6-4e65-a489-d93f2ceeb2b7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018227546Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136672024-02-08T06:44:32.798Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018250424Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136682024-02-08T06:44:32.798Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018270213Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136692024-02-08T06:44:32.798Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018284862Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136702024-02-08T06:44:32.798Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.01830643Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136712024-02-08T06:44:32.798Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018322589Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136722024-02-08T06:44:32.798Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018362266Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136732024-02-08T06:44:32.798Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018385694Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136742024-02-08T06:44:32.798Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018405013Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"io task","client":"0","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136752024-02-08T06:44:32.798Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.01843498Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"io task","client":"1","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136762024-02-08T06:44:32.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018608037Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"io task","client":"0","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136772024-02-08T06:44:32.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018660693Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"io task","client":"1","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136782024-02-08T06:44:32.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.018691681Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136792024-02-08T06:44:32.798Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.019255969Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136802024-02-08T06:44:32.799Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.019322004Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136812024-02-08T06:44:32.799Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.01937507Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136822024-02-08T06:44:32.799Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.019417207Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136832024-02-08T06:44:32.799Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.019866233Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136842024-02-08T06:44:32.799Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.019994143Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136852024-02-08T06:44:32.799Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.02043933Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136862024-02-08T06:44:32.799Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.020472887Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136872024-02-08T06:44:32.799Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.020489316Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136882024-02-08T06:44:32.799Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.02056615Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136892024-02-08T06:44:32.799Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.020594008Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136902024-02-08T06:44:32.799Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.020608987Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136912024-02-08T06:44:32.799Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.020824741Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136922024-02-08T06:44:32.799Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.020852049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136932024-02-08T06:44:32.799Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.021381509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136942024-02-08T06:44:32.806Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.021867253Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136952024-02-08T06:44:32.806Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.02190246Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136962024-02-08T06:44:32.806Z{"msg":"[0]R flush_numbers[0..12]: [209, 199, 200, 209, 209, 168, 209, 192, 172, 206, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.021920279Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136972024-02-08T06:44:32.806Z{"msg":"[0]R generation[0..12]: [125, 121, 122, 125, 125, 104, 125, 116, 107, 124, 123, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.021943787Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136982024-02-08T06:44:32.806Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.021959706Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
136992024-02-08T06:44:32.806Z{"msg":"[1]R flush_numbers[0..12]: [209, 199, 200, 209, 209, 168, 209, 192, 172, 206, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.021980874Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137002024-02-08T06:44:32.806Z{"msg":"[1]R generation[0..12]: [125, 121, 122, 125, 125, 104, 125, 116, 107, 124, 123, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.021995833Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137012024-02-08T06:44:32.806Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022017452Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137022024-02-08T06:44:32.806Z{"msg":"[2]R flush_numbers[0..12]: [209, 199, 200, 209, 209, 168, 209, 192, 172, 206, 202, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.02203179Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137032024-02-08T06:44:32.806Z{"msg":"[2]R generation[0..12]: [125, 121, 122, 125, 125, 104, 125, 116, 107, 124, 123, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022058598Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137042024-02-08T06:44:32.806Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022073837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137052024-02-08T06:44:32.806Z{"msg":"Max found gen is 126","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
137062024-02-08T06:44:32.806Zlevel":30,"time":"2024-02-08T06:44:32.022098745Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137072024-02-08T06:44:32.806Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022113744Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137082024-02-08T06:44:32.807Z{"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022126693Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137092024-02-08T06:44:32.807Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022139832Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137102024-02-08T06:44:32.807Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022154041Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137112024-02-08T06:44:32.807Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.02217403Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137122024-02-08T06:44:32.807Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022188029Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137132024-02-08T06:44:32.807Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022208327Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137142024-02-08T06:44:32.807Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022222596Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137152024-02-08T06:44:32.807Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022242284Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137162024-02-08T06:44:32.807Z{"msg":"d12a31ab-12d6-4e65-a489-d93f2ceeb2b7 is now active with session: 912cb1fe-b51a-4168-bade-907f57902088","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022255463Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137172024-02-08T06:44:32.807Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022274462Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642,"session_id":"912cb1fe-b51a-4168-bade-907f57902088"}
137182024-02-08T06:44:32.807Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:32.022288111Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642}
137192024-02-08T06:44:32.807ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
137202024-02-08T06:44:32.807ZLoading write count information from file "/tmp/test_repair_verify.data"
137212024-02-08T06:44:32.807ZRun Repair workload
137222024-02-08T06:44:32.808Z01/30 Write block 417 len 28672 data: 11 12 12 11 10 10 9
137232024-02-08T06:44:32.808Z02/30 Write block 430 len 12288 data: 8 7 8
137242024-02-08T06:44:32.808Z03/30 Flush
137252024-02-08T06:44:32.808Z04/30 Write block 306 len 4096 data: 13
137262024-02-08T06:44:32.808Z05/30 Write block 106 len 36864 data: 9 8 9 7 3 3 4 7 8
137272024-02-08T06:44:32.808Z06/30 Write block 57 len 20480 data: 12 7 6 7 7
137282024-02-08T06:44:32.808Z07/30 Read block 436 len 20480
137292024-02-08T06:44:32.812Z08/30 Write block 263 len 8192 data: 12 12
137302024-02-08T06:44:32.812Z09/30 Read block 65 len 4096
137312024-02-08T06:44:32.812Z10/30 Read block 408 len 24576
137322024-02-08T06:44:32.812Z11/30 Write block 547 len 16384 data: 15 16 18 16
137332024-02-08T06:44:32.812Z12/30 Read block 412 len 4096
137342024-02-08T06:44:32.812Z13/30 Read block 484 len 36864
137352024-02-08T06:44:32.812Z14/30 Read block 536 len 4096
137362024-02-08T06:44:32.812Z15/30 Write block 337 len 36864 data: 15 16 16 14 14 13 12 11 12
137372024-02-08T06:44:32.812Z16/30 Read block 577 len 28672
137382024-02-08T06:44:32.812Z17/30 Read block 170 len 12288
137392024-02-08T06:44:32.816Z18/30 Read block 263 len 28672
137402024-02-08T06:44:32.816Z19/30 Read block 77 len 8192
137412024-02-08T06:44:32.816Z20/30 Read block 461 len 12288
137422024-02-08T06:44:32.816Z21/30 Read block 279 len 24576
137432024-02-08T06:44:32.816Z22/30 Read block 297 len 20480
137442024-02-08T06:44:32.819Z23/30 Flush
137452024-02-08T06:44:32.819Z24/30 Write block 218 len 20480 data: 11 12 12 13 14
137462024-02-08T06:44:32.819Z25/30 Read block 19 len 16384
137472024-02-08T06:44:32.822Z26/30 Flush
137482024-02-08T06:44:32.825Z27/30 Write block 467 len 32768 data: 9 9 10 9 8 10 9 11
137492024-02-08T06:44:32.825Z28/30 Read block 563 len 16384
137502024-02-08T06:44:32.825Z29/30 Read block 119 len 32768
137512024-02-08T06:44:32.829Z30/30 Read block 19 len 32768
137522024-02-08T06:44:32.829Z----------------------------------------------------------------
137532024-02-08T06:44:32.829Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
137542024-02-08T06:44:32.829ZStates: Active Active Active
137552024-02-08T06:44:32.829ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
137562024-02-08T06:44:32.829Z 1 Acked 1000 Write 7 Done Sent Done false
137572024-02-08T06:44:32.829Z 2 Acked 1001 Write 3 Done Sent Done false
137582024-02-08T06:44:32.830Z 3 Acked 1002 Flush 0 Done Sent Done false
137592024-02-08T06:44:32.830Z 4 Acked 1003 Write 1 Done Sent Done false
137602024-02-08T06:44:32.830Z 5 Acked 1004 Write 9 Done Sent Done false
137612024-02-08T06:44:32.830Z 6 Acked 1005 Write 5 Done Sent Done false
137622024-02-08T06:44:32.830Z 7 Acked 1006 Read 5 Done Sent Done false
137632024-02-08T06:44:32.830Z 8 Acked 1007 Write 2 Done Sent Done false
137642024-02-08T06:44:32.830Z 9 Acked 1008 Read 1 Done Sent Done false
137652024-02-08T06:44:32.830Z 10 Acked 1009 Read 6 Done Sent Done false
137662024-02-08T06:44:32.830Z 11 Acked 1010 Write 4 Done Sent Done false
137672024-02-08T06:44:32.830Z 12 Acked 1011 Read 1 Done Sent Done false
137682024-02-08T06:44:32.830Z 13 Acked 1012 Read 9 Done Sent Done false
137692024-02-08T06:44:32.830Z 14 Acked 1013 Read 1 Done Sent Done false
137702024-02-08T06:44:32.830Z 15 Acked 1014 Write 9 Done Sent Done false
137712024-02-08T06:44:32.830Z 16 Acked 1015 Read 7 Done Sent Done false
137722024-02-08T06:44:32.830Z 17 Acked 1016 Read 3 Done Sent Done false
137732024-02-08T06:44:32.831Z 18 Acked 1017 Read 7 Done Sent Done false
137742024-02-08T06:44:32.831Z 19 Acked 1018 Read 2 Done Sent Done false
137752024-02-08T06:44:32.831Z 20 Acked 1019 Read 3 Done Sent Done false
137762024-02-08T06:44:32.831Z 21 Acked 1020 Read 6 Done Sent Done false
137772024-02-08T06:44:32.831Z 22 Acked 1021 Read 5 Done Sent Done false
137782024-02-08T06:44:32.831Z 23 Acked 1022 Flush 0 Done Sent Done false
137792024-02-08T06:44:32.831Z 24 Acked 1023 Write 5 Done Sent Done false
137802024-02-08T06:44:32.831Z 25 Acked 1024 Read 4 Done Sent Done false
137812024-02-08T06:44:32.831Z 26 Acked 1025 Flush 0 Done Sent Done false
137822024-02-08T06:44:32.831Z 27 Acked 1026 Write 8 Done Sent Done false
137832024-02-08T06:44:32.831Z 28 Acked 1027 Read 4 Done Sent Done false
137842024-02-08T06:44:32.831Z 29 Acked 1028 Read 8 Done Sent Done false
137852024-02-08T06:44:32.831Z 30 Acked 1029 Read 8 Done Sent Done false
137862024-02-08T06:44:32.831Z STATES DS:0 DS:1 DS:2 TOTAL
137872024-02-08T06:44:32.831Z New 0 0 0 0
137882024-02-08T06:44:32.831Z Sent 0 30 0 30
137892024-02-08T06:44:32.832Z Done 30 0 30 60
137902024-02-08T06:44:32.832Z Skipped 0 0 0 0
137912024-02-08T06:44:32.832Z Error 0 0 0 0
137922024-02-08T06:44:32.832ZLast Flush: 1025 0 1025
137932024-02-08T06:44:32.832ZDownstairs last five completed:
137942024-02-08T06:44:32.832ZUpstairs last five completed: 30 29 28 27 26
137952024-02-08T06:44:32.832Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:32.052838758Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1642}
137962024-02-08T06:44:32.832ZWrote out file "/tmp/test_repair_verify.data"
137972024-02-08T06:44:32.836Z
137982024-02-08T06:44:32.836ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
137992024-02-08T06:44:32.848ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138002024-02-08T06:44:32.849Z 0 000-019 125 125 125 209 209 209 F F F
138012024-02-08T06:44:32.849Z 1 020-039 121 121 121 199 199 199 F F F
138022024-02-08T06:44:32.849Z 2 040-059 126 122 126 211 200 211 F F F <---
138032024-02-08T06:44:32.849Z 3 060-079 126 125 126 211 209 211 F F F <---
138042024-02-08T06:44:32.849Z 4 080-099 125 125 125 209 209 209 F F F
138052024-02-08T06:44:32.852Z 5 100-119 126 104 126 211 168 211 F F F <---
138062024-02-08T06:44:32.852Z 6 120-139 125 125 125 209 209 209 F F F
138072024-02-08T06:44:32.853Z 7 140-159 116 116 116 192 192 192 F F F
138082024-02-08T06:44:32.853Z 8 160-179 107 107 107 172 172 172 F F F
138092024-02-08T06:44:32.853Z 9 180-199 124 124 124 206 206 206 F F F
138102024-02-08T06:44:32.853Z 10 200-219 126 123 126 212 202 212 F F F <---
138112024-02-08T06:44:32.853Z 11 220-239 126 121 126 212 199 212 F F F <---
138122024-02-08T06:44:32.853Z 12 240-259 124 124 124 208 208 208 F F F
138132024-02-08T06:44:32.853Z 13 260-279 126 125 126 211 209 211 F F F <---
138142024-02-08T06:44:32.853Z 14 280-299 125 125 125 209 209 209 F F F
138152024-02-08T06:44:32.853Z 15 300-319 126 122 126 211 201 211 F F F <---
138162024-02-08T06:44:32.853Z 16 320-339 126 119 126 211 197 211 F F F <---
138172024-02-08T06:44:32.853Z 17 340-359 126 124 126 211 206 211 F F F <---
138182024-02-08T06:44:32.853Z 18 360-379 124 124 124 206 206 206 F F F
138192024-02-08T06:44:32.853Z 19 380-399 124 124 124 206 206 206 F F F
138202024-02-08T06:44:32.853Z 20 400-419 126 125 126 210 209 210 F F F <---
138212024-02-08T06:44:32.853Z 21 420-439 126 125 126 210 209 210 F F F <---
138222024-02-08T06:44:32.854Z 22 440-459 125 125 125 209 209 209 F F F
138232024-02-08T06:44:32.854Z 23 460-479 120 120 120 198 198 198 T F T <---
138242024-02-08T06:44:32.854Z 24 480-499 111 111 111 181 181 181 F F F
138252024-02-08T06:44:32.854Z 25 500-519 111 111 111 181 181 181 F F F
138262024-02-08T06:44:32.854Z 26 520-539 121 121 121 199 199 199 F F F
138272024-02-08T06:44:32.854Z 27 540-559 126 125 126 211 209 211 F F F <---
138282024-02-08T06:44:32.854Z 28 560-579 124 124 124 206 206 206 F F F
138292024-02-08T06:44:32.854Z 29 580-599 125 125 125 209 209 209 F F F
138302024-02-08T06:44:32.854ZMax gen: 126, Max flush: 212
138312024-02-08T06:44:32.854ZError: Difference in extent metadata found!
138322024-02-08T06:44:32.854ZA Difference in extent metadata is expected here
138332024-02-08T06:44:32.854ZOn loop 63
138342024-02-08T06:44:32.854Z
138352024-02-08T06:44:32.860ZVerifying data now
138362024-02-08T06:44:37.474ZNo change after verify
138372024-02-08T06:44:37.475ZLoop: 63 Downstairs dump after verify (and repair):
138382024-02-08T06:44:37.489ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138392024-02-08T06:44:37.490Z 0 000-019 125 125 125 209 209 209 F F F
138402024-02-08T06:44:37.490Z 1 020-039 121 121 121 199 199 199 F F F
138412024-02-08T06:44:37.490Z 2 040-059 126 126 126 211 211 211 F F F
138422024-02-08T06:44:37.490Z 3 060-079 126 126 126 211 211 211 F F F
138432024-02-08T06:44:37.490Z 4 080-099 125 125 125 209 209 209 F F F
138442024-02-08T06:44:37.490Z 5 100-119 126 126 126 211 211 211 F F F
138452024-02-08T06:44:37.490Z 6 120-139 125 125 125 209 209 209 F F F
138462024-02-08T06:44:37.490Z 7 140-159 116 116 116 192 192 192 F F F
138472024-02-08T06:44:37.490Z 8 160-179 107 107 107 172 172 172 F F F
138482024-02-08T06:44:37.490Z 9 180-199 124 124 124 206 206 206 F F F
138492024-02-08T06:44:37.490Z 10 200-219 126 126 126 212 212 212 F F F
138502024-02-08T06:44:37.496Z 11 220-239 126 126 126 212 212 212 F F F
138512024-02-08T06:44:37.496Z 12 240-259 124 124 124 208 208 208 F F F
138522024-02-08T06:44:37.497Z 13 260-279 126 126 126 211 211 211 F F F
138532024-02-08T06:44:37.497Z 14 280-299 125 125 125 209 209 209 F F F
138542024-02-08T06:44:37.497Z 15 300-319 126 126 126 211 211 211 F F F
138552024-02-08T06:44:37.497Z 16 320-339 126 126 126 211 211 211 F F F
138562024-02-08T06:44:37.497Z 17 340-359 126 126 126 211 211 211 F F F
138572024-02-08T06:44:37.497Z 18 360-379 124 124 124 206 206 206 F F F
138582024-02-08T06:44:37.497Z 19 380-399 124 124 124 206 206 206 F F F
138592024-02-08T06:44:37.497Z 20 400-419 126 126 126 210 210 210 F F F
138602024-02-08T06:44:37.497Z 21 420-439 126 126 126 210 210 210 F F F
138612024-02-08T06:44:37.497Z 22 440-459 125 125 125 209 209 209 F F F
138622024-02-08T06:44:37.497Z 23 460-479 127 127 127 213 213 213 F F F
138632024-02-08T06:44:37.497Z 24 480-499 111 111 111 181 181 181 F F F
138642024-02-08T06:44:37.497Z 25 500-519 111 111 111 181 181 181 F F F
138652024-02-08T06:44:37.497Z 26 520-539 121 121 121 199 199 199 F F F
138662024-02-08T06:44:37.498Z 27 540-559 126 126 126 211 211 211 F F F
138672024-02-08T06:44:37.498Z 28 560-579 124 124 124 206 206 206 F F F
138682024-02-08T06:44:37.498Z 29 580-599 125 125 125 209 209 209 F F F
138692024-02-08T06:44:37.498ZMax gen: 127, Max flush: 213
138702024-02-08T06:44:37.498Z
138712024-02-08T06:44:37.498ZBegin loop 64
138722024-02-08T06:44:37.498ZDownstairs to restart: 0
138732024-02-08T06:44:37.510Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735188156Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650}
138742024-02-08T06:44:37.510Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735417549Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650}
138752024-02-08T06:44:37.510Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735436817Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650}
138762024-02-08T06:44:37.515Z{"msg":"Crucible 9f04672a-bcb2-4f72-849b-d1f1a247aba1 has session id: 4e4941ed-961a-4060-8ccb-9b4a12ceb0cf","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735470755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138772024-02-08T06:44:37.515Z{"msg":"Crucible runtime is spawned
138782024-02-08T06:44:37.515ZUpstairs opts: Upstairs UUID: 9f04672a-bcb2-4f72-849b-d1f1a247aba1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735508752Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138792024-02-08T06:44:37.516Z{"msg":"Crucible stats registered with UUID: 9f04672a-bcb2-4f72-849b-d1f1a247aba1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735547209Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138802024-02-08T06:44:37.516Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735573737Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650}
138812024-02-08T06:44:37.516Z{"msg":"9f04672a-bcb2-4f72-849b-d1f1a247aba1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735600855Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138822024-02-08T06:44:37.516Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735614044Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138832024-02-08T06:44:37.516Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735634523Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138842024-02-08T06:44:37.516Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735648781Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138852024-02-08T06:44:37.516Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.73566338Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138862024-02-08T06:44:37.516Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735680379Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138872024-02-08T06:44:37.516Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735701807Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138882024-02-08T06:44:37.516Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735723656Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138892024-02-08T06:44:37.516Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735748534Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"io task","client":"2","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138902024-02-08T06:44:37.516Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735810769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138912024-02-08T06:44:37.516Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.735899592Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"io task","client":"2","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138922024-02-08T06:44:37.516Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:36.735943009Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138932024-02-08T06:44:37.516Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:36.735973777Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138942024-02-08T06:44:37.517Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:36.736002955Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138952024-02-08T06:44:37.517Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.736039722Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138962024-02-08T06:44:37.517Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.736088928Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138972024-02-08T06:44:37.517Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.736131525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138982024-02-08T06:44:37.517Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.736171122Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
138992024-02-08T06:44:37.517Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.736579052Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139002024-02-08T06:44:37.517Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.736619129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139012024-02-08T06:44:37.517Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.736645067Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139022024-02-08T06:44:37.517Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.736680114Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139032024-02-08T06:44:37.517Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.737195705Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139042024-02-08T06:44:37.517Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.737242962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139052024-02-08T06:44:37.517Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.737739574Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139062024-02-08T06:44:37.517Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.737787441Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139072024-02-08T06:44:37.517Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.737811919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139082024-02-08T06:44:37.517Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.737834777Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139092024-02-08T06:44:37.518Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.737870564Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139102024-02-08T06:44:37.518Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:36.737900162Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139112024-02-08T06:44:38.512Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.737400452Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139122024-02-08T06:44:38.518Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.737773464Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139132024-02-08T06:44:38.518Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.738385788Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139142024-02-08T06:44:38.518Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.738422635Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139152024-02-08T06:44:38.518Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.738869062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139162024-02-08T06:44:38.519Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739370384Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139172024-02-08T06:44:38.519Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739417621Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139182024-02-08T06:44:38.519Z{"msg":"[0]R flush_numbers[0..12]: [209, 199, 211, 211, 209, 211, 209, 192, 172, 206, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739438109Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139192024-02-08T06:44:38.519Z{"msg":"[0]R generation[0..12]: [125, 121, 126, 126, 125, 126, 125, 116, 107, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739454878Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139202024-02-08T06:44:38.519Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739469557Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139212024-02-08T06:44:38.519Z{"msg":"[1]R flush_numbers[0..12]: [209, 199, 211, 211, 209, 211, 209, 192, 172, 206, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739492705Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139222024-02-08T06:44:38.519Z{"msg":"[1]R generation[0..12]: [125, 121, 126, 126, 125, 126, 125, 116, 107, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739507014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139232024-02-08T06:44:38.519Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739521823Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139242024-02-08T06:44:38.519Z{"msg":"[2]R flush_numbers[0..12]: [209, 199, 211, 211, 209, 211, 209, 192, 172, 206, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739536032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
139252024-02-08T06:44:38.519Z,"":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139262024-02-08T06:44:38.519Z{"msg":"[2]R generation[0..12]: [125, 121, 126, 126, 125, 126, 125, 116, 107, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.73956461Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139272024-02-08T06:44:38.519Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739579158Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139282024-02-08T06:44:38.519Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739599517Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139292024-02-08T06:44:38.519Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739613646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139302024-02-08T06:44:38.519Z{"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739633934Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139312024-02-08T06:44:38.520Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739647783Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139322024-02-08T06:44:38.520Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739667612Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139332024-02-08T06:44:38.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739680131Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139342024-02-08T06:44:38.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.7396941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139352024-02-08T06:44:38.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739708169Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139362024-02-08T06:44:38.520Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739721638Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139372024-02-08T06:44:38.520Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739740666Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139382024-02-08T06:44:38.520Z{"msg":"9f04672a-bcb2-4f72-849b-d1f1a247aba1 is now active with session: 4e4941ed-961a-4060-8ccb-9b4a12ceb0cf","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739753675Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139392024-02-08T06:44:38.520Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739772174Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650,"session_id":"4e4941ed-961a-4060-8ccb-9b4a12ceb0cf"}
139402024-02-08T06:44:38.520Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:37.739784683Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650}
139412024-02-08T06:44:38.520ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
139422024-02-08T06:44:38.520ZLoading write count information from file "/tmp/test_repair_verify.data"
139432024-02-08T06:44:38.520ZRun Repair workload
139442024-02-08T06:44:38.520Z01/30 Write block 32 len 4096 data: 10
139452024-02-08T06:44:38.524Z02/30 Read block 363 len 36864
139462024-02-08T06:44:38.524Z03/30 Write block 460 len 24576 data: 9 10 11 11 9 8
139472024-02-08T06:44:38.524Z04/30 Write block 44 len 4096 data: 5
139482024-02-08T06:44:38.524Z05/30 Write block 53 len 28672 data: 8 10 10 13 13 8 7
139492024-02-08T06:44:38.524Z06/30 Flush
139502024-02-08T06:44:38.524Z07/30 Write block 314 len 20480 data: 9 5 7 7 9
139512024-02-08T06:44:38.524Z08/30 Write block 591 len 4096 data: 10
139522024-02-08T06:44:38.524Z09/30 Read block 321 len 32768
139532024-02-08T06:44:38.528Z10/30 Write block 524 len 24576 data: 8 8 8 8 10 10
139542024-02-08T06:44:38.528Z11/30 Flush
139552024-02-08T06:44:38.528Z12/30 Write block 94 len 36864 data: 16 16 16 16 14 14 15 13 11
139562024-02-08T06:44:38.531Z13/30 Write block 349 len 28672 data: 7 8 9 8 7 8 8
139572024-02-08T06:44:38.531Z14/30 Flush
139582024-02-08T06:44:38.534Z15/30 Write block 46 len 20480 data: 6 4 5 4 7
139592024-02-08T06:44:38.534Z16/30 Read block 138 len 40960
139602024-02-08T06:44:38.537Z17/30 Write block 364 len 28672 data: 8 8 6 8 8 10 10
139612024-02-08T06:44:38.537Z18/30 Read block 280 len 36864
139622024-02-08T06:44:38.537Z19/30 Read block 475 len 16384
139632024-02-08T06:44:38.537Z20/30 Read block 352 len 8192
139642024-02-08T06:44:38.540Z21/30 Write block 362 len 24576 data: 6 5 9 9 7 9
139652024-02-08T06:44:38.540Z22/30 Write block 439 len 40960 data: 9 10 10 9 10 10 11 9 10 11
139662024-02-08T06:44:38.540Z23/30 Read block 313 len 28672
139672024-02-08T06:44:38.543Z24/30 Read block 211 len 24576
139682024-02-08T06:44:38.543Z25/30 Flush
139692024-02-08T06:44:38.546Z26/30 Flush
139702024-02-08T06:44:38.546Z27/30 Write block 520 len 28672 data: 9 7 7 8 9 9 9
139712024-02-08T06:44:38.546Z28/30 Read block 255 len 20480
139722024-02-08T06:44:38.550Z29/30 Write block 496 len 4096 data: 7
139732024-02-08T06:44:38.550Z30/30 Read block 333 len 28672
139742024-02-08T06:44:38.550Z----------------------------------------------------------------
139752024-02-08T06:44:38.550Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
139762024-02-08T06:44:38.550ZStates: Active Active Active
139772024-02-08T06:44:38.550ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
139782024-02-08T06:44:38.550Z 1 Acked 1000 Write 1 Done Done Done false
139792024-02-08T06:44:38.550Z 2 Acked 1001 Read 9 Sent Done Done false
139802024-02-08T06:44:38.550Z 3 Acked 1002 Write 6 Sent Done Done false
139812024-02-08T06:44:38.550Z 4 Acked 1003 Write 1 Sent Done Done false
139822024-02-08T06:44:38.551Z 5 Acked 1004 Write 7 Sent Done Done false
139832024-02-08T06:44:38.551Z 6 Acked 1005 Flush 0 Sent Done Done false
139842024-02-08T06:44:38.551Z 7 Acked 1006 Write 5 Sent Done Done false
139852024-02-08T06:44:38.551Z 8 Acked 1007 Write 1 Sent Done Done false
139862024-02-08T06:44:38.551Z 9 Acked 1008 Read 8 Sent Done Done false
139872024-02-08T06:44:38.551Z 10 Acked 1009 Write 6 Sent Done Done false
139882024-02-08T06:44:38.551Z 11 Acked 1010 Flush 0 Sent Done Done false
139892024-02-08T06:44:38.551Z 12 Acked 1011 Write 9 Sent Done Done false
139902024-02-08T06:44:38.551Z 13 Acked 1012 Write 7 Sent Done Done false
139912024-02-08T06:44:38.551Z 14 Acked 1013 Flush 0 Sent Done Done false
139922024-02-08T06:44:38.551Z 15 Acked 1014 Write 5 Sent Done Done false
139932024-02-08T06:44:38.551Z 16 Acked 1015 Read 10 Sent Done Done false
139942024-02-08T06:44:38.551Z 17 Acked 1016 Write 7 Sent Done Done false
139952024-02-08T06:44:38.551Z 18 Acked 1017 Read 9 Sent Done Done false
139962024-02-08T06:44:38.551Z 19 Acked 1018 Read 4 Sent Done Done false
139972024-02-08T06:44:38.552Z 20 Acked 1019 Read 2 Sent Done Done false
139982024-02-08T06:44:38.552Z 21 Acked 1020 Write 6 Sent Done Done false
139992024-02-08T06:44:38.552Z 22 Acked 1021 Write 10 Sent Done Done false
140002024-02-08T06:44:38.552Z 23 Acked 1022 Read 7 Sent Done Done false
140012024-02-08T06:44:38.552Z 24 Acked 1023 Read 6 Sent Done Done false
140022024-02-08T06:44:38.552Z 25 Acked 1024 Flush 0 Sent Done Done false
140032024-02-08T06:44:38.552Z 26 Acked 1025 Flush 0 Sent Done Done false
140042024-02-08T06:44:38.552Z 27 Acked 1026 Write 7 Sent Done Done false
140052024-02-08T06:44:38.552Z 28 Acked 1027 Read 5 Sent Done Done false
140062024-02-08T06:44:38.552Z 29 Acked 1028 Write 1 Sent Done Done false
140072024-02-08T06:44:38.552Z 30 Acked 1029 Read 7 Sent Sent Done false
140082024-02-08T06:44:38.552Z STATES DS:0 DS:1 DS:2 TOTAL
140092024-02-08T06:44:38.552Z New 0 0 0 0
140102024-02-08T06:44:38.552Z Sent 29 1 0 30
140112024-02-08T06:44:38.553Z Done 1 29 30 60
140122024-02-08T06:44:38.553Z Skipped 0 0 0 0
140132024-02-08T06:44:38.553Z Error 0 0 0 0
140142024-02-08T06:44:38.553ZLast Flush: 0 1025 1025
140152024-02-08T06:44:38.553ZDownstairs last five completed:
140162024-02-08T06:44:38.553ZUpstairs last five completed: 30 29 28 27 26
140172024-02-08T06:44:38.553Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:37.774567442Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1650}
140182024-02-08T06:44:38.553ZWrote out file "/tmp/test_repair_verify.data"
140192024-02-08T06:44:38.556Z
140202024-02-08T06:44:38.556ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
140212024-02-08T06:44:38.572ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140222024-02-08T06:44:38.572Z 0 000-019 125 125 125 209 209 209 F F F
140232024-02-08T06:44:38.573Z 1 020-039 121 128 128 199 214 214 T F F <---
140242024-02-08T06:44:38.573Z 2 040-059 126 128 128 211 217 217 F F F <---
140252024-02-08T06:44:38.573Z 3 060-079 126 126 126 211 211 211 F F F
140262024-02-08T06:44:38.573Z 4 080-099 125 128 128 209 216 216 F F F <---
140272024-02-08T06:44:38.573Z 5 100-119 126 128 128 211 216 216 F F F <---
140282024-02-08T06:44:38.573Z 6 120-139 125 125 125 209 209 209 F F F
140292024-02-08T06:44:38.573Z 7 140-159 116 116 116 192 192 192 F F F
140302024-02-08T06:44:38.573Z 8 160-179 107 107 107 172 172 172 F F F
140312024-02-08T06:44:38.573Z 9 180-199 124 124 124 206 206 206 F F F
140322024-02-08T06:44:38.573Z 10 200-219 126 126 126 212 212 212 F F F
140332024-02-08T06:44:38.577Z 11 220-239 126 126 126 212 212 212 F F F
140342024-02-08T06:44:38.577Z 12 240-259 124 124 124 208 208 208 F F F
140352024-02-08T06:44:38.577Z 13 260-279 126 126 126 211 211 211 F F F
140362024-02-08T06:44:38.577Z 14 280-299 125 125 125 209 209 209 F F F
140372024-02-08T06:44:38.577Z 15 300-319 126 128 128 211 215 215 F F F <---
140382024-02-08T06:44:38.577Z 16 320-339 126 126 126 211 211 211 F F F
140392024-02-08T06:44:38.577Z 17 340-359 126 128 128 211 216 216 F F F <---
140402024-02-08T06:44:38.577Z 18 360-379 124 128 128 206 217 217 F F F <---
140412024-02-08T06:44:38.577Z 19 380-399 124 124 124 206 206 206 F F F
140422024-02-08T06:44:38.577Z 20 400-419 126 126 126 210 210 210 F F F
140432024-02-08T06:44:38.577Z 21 420-439 126 128 128 210 217 217 F F F <---
140442024-02-08T06:44:38.577Z 22 440-459 125 128 128 209 217 217 F F F <---
140452024-02-08T06:44:38.578Z 23 460-479 127 128 128 213 214 214 F F F <---
140462024-02-08T06:44:38.578Z 24 480-499 111 111 111 181 181 181 F T T <---
140472024-02-08T06:44:38.578Z 25 500-519 111 111 111 181 181 181 F F F
140482024-02-08T06:44:38.578Z 26 520-539 121 128 128 199 215 215 F T T <---
140492024-02-08T06:44:38.578Z 27 540-559 126 126 126 211 211 211 F F F
140502024-02-08T06:44:38.578Z 28 560-579 124 124 124 206 206 206 F F F
140512024-02-08T06:44:38.578Z 29 580-599 125 128 128 209 215 215 F F F <---
140522024-02-08T06:44:38.578ZMax gen: 128, Max flush: 217
140532024-02-08T06:44:38.578ZError: Difference in extent metadata found!
140542024-02-08T06:44:38.578ZA Difference in extent metadata is expected here
140552024-02-08T06:44:38.578ZOn loop 64
140562024-02-08T06:44:38.578Z
140572024-02-08T06:44:38.585ZVerifying data now
140582024-02-08T06:44:44.210ZNo change after verify
140592024-02-08T06:44:44.210ZLoop: 64 Downstairs dump after verify (and repair):
140602024-02-08T06:44:44.225ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140612024-02-08T06:44:44.225Z 0 000-019 125 125 125 209 209 209 F F F
140622024-02-08T06:44:44.225Z 1 020-039 128 128 128 214 214 214 F F F
140632024-02-08T06:44:44.225Z 2 040-059 128 128 128 217 217 217 F F F
140642024-02-08T06:44:44.225Z 3 060-079 126 126 126 211 211 211 F F F
140652024-02-08T06:44:44.225Z 4 080-099 128 128 128 216 216 216 F F F
140662024-02-08T06:44:44.225Z 5 100-119 128 128 128 216 216 216 F F F
140672024-02-08T06:44:44.229Z 6 120-139 125 125 125 209 209 209 F F F
140682024-02-08T06:44:44.229Z 7 140-159 116 116 116 192 192 192 F F F
140692024-02-08T06:44:44.229Z 8 160-179 107 107 107 172 172 172 F F F
140702024-02-08T06:44:44.229Z 9 180-199 124 124 124 206 206 206 F F F
140712024-02-08T06:44:44.229Z 10 200-219 126 126 126 212 212 212 F F F
140722024-02-08T06:44:44.229Z 11 220-239 126 126 126 212 212 212 F F F
140732024-02-08T06:44:44.229Z 12 240-259 124 124 124 208 208 208 F F F
140742024-02-08T06:44:44.229Z 13 260-279 126 126 126 211 211 211 F F F
140752024-02-08T06:44:44.229Z 14 280-299 125 125 125 209 209 209 F F F
140762024-02-08T06:44:44.229Z 15 300-319 128 128 128 215 215 215 F F F
140772024-02-08T06:44:44.229Z 16 320-339 126 126 126 211 211 211 F F F
140782024-02-08T06:44:44.229Z 17 340-359 128 128 128 216 216 216 F F F
140792024-02-08T06:44:44.230Z 18 360-379 128 128 128 217 217 217 F F F
140802024-02-08T06:44:44.230Z 19 380-399 124 124 124 206 206 206 F F F
140812024-02-08T06:44:44.230Z 20 400-419 126 126 126 210 210 210 F F F
140822024-02-08T06:44:44.230Z 21 420-439 128 128 128 217 217 217 F F F
140832024-02-08T06:44:44.230Z 22 440-459 128 128 128 217 217 217 F F F
140842024-02-08T06:44:44.230Z 23 460-479 128 128 128 214 214 214 F F F
140852024-02-08T06:44:44.230Z 24 480-499 129 129 129 218 218 218 F F F
140862024-02-08T06:44:44.230Z 25 500-519 111 111 111 181 181 181 F F F
140872024-02-08T06:44:44.230Z 26 520-539 129 129 129 218 218 218 F F F
140882024-02-08T06:44:44.230Z 27 540-559 126 126 126 211 211 211 F F F
140892024-02-08T06:44:44.230Z 28 560-579 124 124 124 206 206 206 F F F
140902024-02-08T06:44:44.230Z 29 580-599 128 128 128 215 215 215 F F F
140912024-02-08T06:44:44.230ZMax gen: 129, Max flush: 218
140922024-02-08T06:44:44.230Z
140932024-02-08T06:44:44.230ZBegin loop 65
140942024-02-08T06:44:44.231ZDownstairs to restart: 2
140952024-02-08T06:44:44.253Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.470361663Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658}
140962024-02-08T06:44:44.253Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.470642182Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658}
140972024-02-08T06:44:44.253ZCrucible runtime is spawned
140982024-02-08T06:44:44.262Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.470712817Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658}
140992024-02-08T06:44:44.263Z{"msg":"Crucible ce5e7a30-1a10-4cb6-8c04-dc76cf9630e1 has session id: 428e561f-3f64-4d71-a762-977d3bce0531","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.470763313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141002024-02-08T06:44:44.263Z{"msg":"Upstairs opts: Upstairs UUID: ce5e7a30-1a10-4cb6-8c04-dc76cf9630e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.47080523Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141012024-02-08T06:44:44.263Z{"msg":"Crucible stats registered with UUID: ce5e7a30-1a10-4cb6-8c04-dc76cf9630e1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.470862005Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141022024-02-08T06:44:44.263Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.470900722Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658}
141032024-02-08T06:44:44.263Z{"msg":"ce5e7a30-1a10-4cb6-8c04-dc76cf9630e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471110167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141042024-02-08T06:44:44.263Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471189601Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141052024-02-08T06:44:44.263Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471247856Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141062024-02-08T06:44:44.263Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471299712Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141072024-02-08T06:44:44.263Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471319881Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141082024-02-08T06:44:44.263Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.47134004Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141092024-02-08T06:44:44.263Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471418854Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141102024-02-08T06:44:44.263Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471503597Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141112024-02-08T06:44:44.263Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471702902Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141122024-02-08T06:44:44.263Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471758388Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141132024-02-08T06:44:44.264Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471787546Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141142024-02-08T06:44:44.264Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.471831273Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141152024-02-08T06:44:44.264Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.47186494Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141162024-02-08T06:44:44.264Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.472352233Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141172024-02-08T06:44:44.264Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.472405429Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141182024-02-08T06:44:44.264Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.472457226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141192024-02-08T06:44:44.264Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.472493523Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141202024-02-08T06:44:44.264Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.473099757Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141212024-02-08T06:44:44.264Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.473152233Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141222024-02-08T06:44:44.264Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.473816804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141232024-02-08T06:44:44.264Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.473854211Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141242024-02-08T06:44:44.264Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.473893738Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141252024-02-08T06:44:44.264Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.473915516Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141262024-02-08T06:44:44.264Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.473944324Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141272024-02-08T06:44:44.265Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.473958793Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141282024-02-08T06:44:44.265Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.474043437Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141292024-02-08T06:44:44.265Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.474080394Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141302024-02-08T06:44:44.265Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.474566767Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141312024-02-08T06:44:44.265Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475054441Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141322024-02-08T06:44:44.265Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475089678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141332024-02-08T06:44:44.265Z{"msg":"[0]R flush_numbers[0..12]: [209, 214, 217, 211, 216, 216, 209, 192, 172, 206, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475105857Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141342024-02-08T06:44:44.265Z{"msg":"[0]R generation[0..12]: [125, 128, 128, 126, 128, 128, 125, 116, 107, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475127405Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141352024-02-08T06:44:44.265Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475142304Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141362024-02-08T06:44:44.265Z{"msg":"[1]R flush_numbers[0..12]: [209, 214, 217, 211, 216, 216, 209, 192, 172, 206, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475163133Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141372024-02-08T06:44:44.265Z{"msg":"[1]R generation[0..12]: [125, 128, 128, 126, 128, 128, 125, 116, 107, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475177381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141382024-02-08T06:44:44.265Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.47519827Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141392024-02-08T06:44:44.265Z{"msg":"[2]R flush_numbers[0..12]: [209, 214, 217, 211, 216, 216, 209, 192, 172, 206, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475212719Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141402024-02-08T06:44:44.265Z{"Wait for a query_work_queue command to finish before sending IO
141412024-02-08T06:44:44.265Zmsg":"[2]R generation[0..12]: [125, 128, 128, 126, 128, 128, 125, 116, 107, 124, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475236407Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141422024-02-08T06:44:44.265Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475251426Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141432024-02-08T06:44:44.266Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475271214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141442024-02-08T06:44:44.266Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475285363Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141452024-02-08T06:44:44.266Z{"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475305182Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141462024-02-08T06:44:44.266Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475319231Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141472024-02-08T06:44:44.266Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475339769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141482024-02-08T06:44:44.266Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475352388Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141492024-02-08T06:44:44.266Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475366497Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141502024-02-08T06:44:44.266Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475379816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141512024-02-08T06:44:44.266Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475393285Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141522024-02-08T06:44:44.266Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475414194Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141532024-02-08T06:44:44.266Z{"msg":"ce5e7a30-1a10-4cb6-8c04-dc76cf9630e1 is now active with session: 428e561f-3f64-4d71-a762-977d3bce0531","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475427733Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141542024-02-08T06:44:44.266Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475440432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658,"session_id":"428e561f-3f64-4d71-a762-977d3bce0531"}
141552024-02-08T06:44:44.266Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:43.475452901Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658}
141562024-02-08T06:44:44.266ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
141572024-02-08T06:44:44.266ZLoading write count information from file "/tmp/test_repair_verify.data"
141582024-02-08T06:44:44.267ZRun Repair workload
141592024-02-08T06:44:44.267Z01/30 Write block 541 len 12288 data: 8 7 9
141602024-02-08T06:44:44.267Z02/30 Write block 155 len 36864 data: 11 12 10 11 10 8 9 11 11
141612024-02-08T06:44:44.267Z03/30 Read block 341 len 32768
141622024-02-08T06:44:44.267Z04/30 Write block 20 len 4096 data: 11
141632024-02-08T06:44:44.267Z05/30 Read block 227 len 20480
141642024-02-08T06:44:44.267Z06/30 Read block 385 len 28672
141652024-02-08T06:44:44.267Z07/30 Write block 181 len 24576 data: 12 13 9 9 12 12
141662024-02-08T06:44:44.267Z08/30 Read block 490 len 8192
141672024-02-08T06:44:44.274Z09/30 Write block 568 len 40960 data: 10 9 11 12 11 11 9 9 9 10
141682024-02-08T06:44:44.274Z10/30 Write block 571 len 12288 data: 13 12 12
141692024-02-08T06:44:44.275Z11/30 Write block 430 len 36864 data: 9 8 9 6 9 10 12 14 12
141702024-02-08T06:44:44.275Z12/30 Write block 288 len 28672 data: 9 8 12 13 12 10 8
141712024-02-08T06:44:44.275Z13/30 Write block 0 len 28672 data: 3 3 3 5 4 5 6
141722024-02-08T06:44:44.275Z14/30 Write block 325 len 40960 data: 13 11 8 6 6 6 10 11 13 16
141732024-02-08T06:44:44.275Z15/30 Flush
141742024-02-08T06:44:44.275Z16/30 Write block 371 len 40960 data: 10 13 12 5 7 9 8 8 8 8
141752024-02-08T06:44:44.275Z17/30 Write block 262 len 36864 data: 14 13 13 13 13 14 14 11 8
141762024-02-08T06:44:44.275Z18/30 Read block 412 len 20480
141772024-02-08T06:44:44.281Z19/30 Write block 568 len 36864 data: 11 10 12 14 13 13 10 10 10
141782024-02-08T06:44:44.281Z20/30 Write block 306 len 28672 data: 14 10 8 8 8 7 8
141792024-02-08T06:44:44.281Z21/30 Write block 107 len 28672 data: 9 10 8 4 4 5 8
141802024-02-08T06:44:44.281Z22/30 Read block 146 len 20480
141812024-02-08T06:44:44.281Z23/30 Read block 164 len 8192
141822024-02-08T06:44:44.281Z24/30 Write block 584 len 12288 data: 10 10 9
141832024-02-08T06:44:44.281Z25/30 Read block 542 len 24576
141842024-02-08T06:44:44.281Z26/30 Write block 219 len 36864 data: 13 13 14 15 14 15 14 14 14
141852024-02-08T06:44:44.281Z27/30 Write block 367 len 4096 data: 10
141862024-02-08T06:44:44.281Z28/30 Write block 68 len 12288 data: 11 12 7
141872024-02-08T06:44:44.281Z29/30 Read block 332 len 20480
141882024-02-08T06:44:44.285Z30/30 Read block 531 len 32768
141892024-02-08T06:44:44.285Z----------------------------------------------------------------
141902024-02-08T06:44:44.285Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:15
141912024-02-08T06:44:44.285ZStates: Active Active Active
141922024-02-08T06:44:44.286ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
141932024-02-08T06:44:44.286Z 16 Acked 1015 Write 10 Done Done Done false
141942024-02-08T06:44:44.286Z 17 Acked 1016 Write 9 Done Done Sent false
141952024-02-08T06:44:44.286Z 18 Acked 1017 Read 5 Done Done Sent false
141962024-02-08T06:44:44.286Z 19 Acked 1018 Write 9 Done Done Sent false
141972024-02-08T06:44:44.286Z 20 Acked 1019 Write 7 Done Done Sent false
141982024-02-08T06:44:44.286Z 21 Acked 1020 Write 7 Done Done Sent false
141992024-02-08T06:44:44.286Z 22 Acked 1021 Read 5 Done Done Sent false
142002024-02-08T06:44:44.286Z 23 Acked 1022 Read 2 Done Done Sent false
142012024-02-08T06:44:44.286Z 24 Acked 1023 Write 3 Done Done Sent false
142022024-02-08T06:44:44.286Z 25 Acked 1024 Read 6 Done Done Sent false
142032024-02-08T06:44:44.286Z 26 Acked 1025 Write 9 Done Done Sent false
142042024-02-08T06:44:44.286Z 27 Acked 1026 Write 1 Done Done Sent false
142052024-02-08T06:44:44.286Z 28 Acked 1027 Write 3 Done Done Sent false
142062024-02-08T06:44:44.286Z 29 Acked 1028 Read 5 Done Done Sent false
142072024-02-08T06:44:44.287Z 30 Acked 1029 Read 8 Done Sent Sent false
142082024-02-08T06:44:44.287Z STATES DS:0 DS:1 DS:2 TOTAL
142092024-02-08T06:44:44.287Z New 0 0 0 0
142102024-02-08T06:44:44.287Z Sent 0 1 14 15
142112024-02-08T06:44:44.287Z Done 15 14 1 30
142122024-02-08T06:44:44.287Z Skipped 0 0 0 0
142132024-02-08T06:44:44.287Z Error 0 0 0 0
142142024-02-08T06:44:44.287ZLast Flush: 1014 1014 1014
142152024-02-08T06:44:44.287ZDownstairs last five completed: 1014 1013 1012 1011 1010
142162024-02-08T06:44:44.287ZUpstairs last five completed: 30 29 28 27 26
142172024-02-08T06:44:44.287Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:43.507196248Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1658}
142182024-02-08T06:44:44.287ZWrote out file "/tmp/test_repair_verify.data"
142192024-02-08T06:44:44.291Z
142202024-02-08T06:44:44.296ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
142212024-02-08T06:44:44.311ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142222024-02-08T06:44:44.311Z 0 000-019 130 130 130 219 219 219 F F F
142232024-02-08T06:44:44.311Z 1 020-039 130 130 130 219 219 219 F F F
142242024-02-08T06:44:44.311Z 2 040-059 128 128 128 217 217 217 F F F
142252024-02-08T06:44:44.311Z 3 060-079 126 126 126 211 211 211 T T F <---
142262024-02-08T06:44:44.311Z 4 080-099 128 128 128 216 216 216 F F F
142272024-02-08T06:44:44.311Z 5 100-119 128 128 128 216 216 216 T T F <---
142282024-02-08T06:44:44.311Z 6 120-139 125 125 125 209 209 209 F F F
142292024-02-08T06:44:44.311Z 7 140-159 130 130 130 219 219 219 F F F
142302024-02-08T06:44:44.311Z 8 160-179 130 130 130 219 219 219 F F F
142312024-02-08T06:44:44.311Z 9 180-199 130 130 130 219 219 219 F F F
142322024-02-08T06:44:44.311Z 10 200-219 126 126 126 212 212 212 T T F <---
142332024-02-08T06:44:44.318Z 11 220-239 126 126 126 212 212 212 T T F <---
142342024-02-08T06:44:44.318Z 12 240-259 124 124 124 208 208 208 F F F
142352024-02-08T06:44:44.318Z 13 260-279 126 126 126 211 211 211 T T F <---
142362024-02-08T06:44:44.319Z 14 280-299 130 130 130 219 219 219 F F F
142372024-02-08T06:44:44.319Z 15 300-319 128 128 128 215 215 215 T T F <---
142382024-02-08T06:44:44.319Z 16 320-339 130 130 130 219 219 219 F F F
142392024-02-08T06:44:44.319Z 17 340-359 128 128 128 216 216 216 F F F
142402024-02-08T06:44:44.320Z 18 360-379 128 128 128 217 217 217 T T T
142412024-02-08T06:44:44.320Z 19 380-399 124 124 124 206 206 206 T T T
142422024-02-08T06:44:44.320Z 20 400-419 126 126 126 210 210 210 F F F
142432024-02-08T06:44:44.320Z 21 420-439 130 130 130 219 219 219 F F F
142442024-02-08T06:44:44.320Z 22 440-459 128 128 128 217 217 217 F F F
142452024-02-08T06:44:44.321Z 23 460-479 128 128 128 214 214 214 F F F
142462024-02-08T06:44:44.321Z 24 480-499 129 129 129 218 218 218 F F F
142472024-02-08T06:44:44.321Z 25 500-519 111 111 111 181 181 181 F F F
142482024-02-08T06:44:44.321Z 26 520-539 129 129 129 218 218 218 F F F
142492024-02-08T06:44:44.321Z 27 540-559 130 130 130 219 219 219 F F F
142502024-02-08T06:44:44.321Z 28 560-579 130 130 130 219 219 219 T T F <---
142512024-02-08T06:44:44.321Z 29 580-599 128 128 128 215 215 215 T T F <---
142522024-02-08T06:44:44.321ZMax gen: 130, Max flush: 219
142532024-02-08T06:44:44.321ZError: Difference in extent metadata found!
142542024-02-08T06:44:44.322ZA Difference in extent metadata is expected here
142552024-02-08T06:44:44.322ZOn loop 65
142562024-02-08T06:44:44.322Z
142572024-02-08T06:44:44.334ZVerifying data now
142582024-02-08T06:44:49.857ZNo change after verify
142592024-02-08T06:44:49.858ZLoop: 65 Downstairs dump after verify (and repair):
142602024-02-08T06:44:49.872ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142612024-02-08T06:44:49.872Z 0 000-019 130 130 130 219 219 219 F F F
142622024-02-08T06:44:49.872Z 1 020-039 130 130 130 219 219 219 F F F
142632024-02-08T06:44:49.872Z 2 040-059 128 128 128 217 217 217 F F F
142642024-02-08T06:44:49.872Z 3 060-079 131 131 131 220 220 220 F F F
142652024-02-08T06:44:49.872Z 4 080-099 128 128 128 216 216 216 F F F
142662024-02-08T06:44:49.873Z 5 100-119 131 131 131 220 220 220 F F F
142672024-02-08T06:44:49.873Z 6 120-139 125 125 125 209 209 209 F F F
142682024-02-08T06:44:49.873Z 7 140-159 130 130 130 219 219 219 F F F
142692024-02-08T06:44:49.873Z 8 160-179 130 130 130 219 219 219 F F F
142702024-02-08T06:44:49.876Z 9 180-199 130 130 130 219 219 219 F F F
142712024-02-08T06:44:49.876Z 10 200-219 131 131 131 220 220 220 F F F
142722024-02-08T06:44:49.876Z 11 220-239 131 131 131 220 220 220 F F F
142732024-02-08T06:44:49.876Z 12 240-259 124 124 124 208 208 208 F F F
142742024-02-08T06:44:49.876Z 13 260-279 131 131 131 220 220 220 F F F
142752024-02-08T06:44:49.877Z 14 280-299 130 130 130 219 219 219 F F F
142762024-02-08T06:44:49.877Z 15 300-319 131 131 131 220 220 220 F F F
142772024-02-08T06:44:49.877Z 16 320-339 130 130 130 219 219 219 F F F
142782024-02-08T06:44:49.877Z 17 340-359 128 128 128 216 216 216 F F F
142792024-02-08T06:44:49.877Z 18 360-379 131 131 131 220 220 220 F F F
142802024-02-08T06:44:49.877Z 19 380-399 131 131 131 220 220 220 F F F
142812024-02-08T06:44:49.877Z 20 400-419 126 126 126 210 210 210 F F F
142822024-02-08T06:44:49.877Z 21 420-439 130 130 130 219 219 219 F F F
142832024-02-08T06:44:49.877Z 22 440-459 128 128 128 217 217 217 F F F
142842024-02-08T06:44:49.877Z 23 460-479 128 128 128 214 214 214 F F F
142852024-02-08T06:44:49.877Z 24 480-499 129 129 129 218 218 218 F F F
142862024-02-08T06:44:49.877Z 25 500-519 111 111 111 181 181 181 F F F
142872024-02-08T06:44:49.877Z 26 520-539 129 129 129 218 218 218 F F F
142882024-02-08T06:44:49.877Z 27 540-559 130 130 130 219 219 219 F F F
142892024-02-08T06:44:49.877Z 28 560-579 131 131 131 220 220 220 F F F
142902024-02-08T06:44:49.877Z 29 580-599 131 131 131 220 220 220 F F F
142912024-02-08T06:44:49.878ZMax gen: 131, Max flush: 220
142922024-02-08T06:44:49.878Z
142932024-02-08T06:44:49.878ZBegin loop 66
142942024-02-08T06:44:49.878ZDownstairs to restart: 1
142952024-02-08T06:44:49.893Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117041382Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666}
142962024-02-08T06:44:49.893Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\nCrucible runtime is spawned
142972024-02-08T06:44:49.898Z rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117346159Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666}
142982024-02-08T06:44:49.898Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117391396Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666}
142992024-02-08T06:44:49.898Z{"msg":"Crucible acdba2ed-3906-411a-b598-7c403dee369a has session id: 88c7aec8-cacb-4b0d-94f4-2009c6625d77","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117433083Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143002024-02-08T06:44:49.898Z{"msg":"Upstairs opts: Upstairs UUID: acdba2ed-3906-411a-b598-7c403dee369a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117488609Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143012024-02-08T06:44:49.898Z{"msg":"Crucible stats registered with UUID: acdba2ed-3906-411a-b598-7c403dee369a","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117527526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143022024-02-08T06:44:49.898Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117569683Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666}
143032024-02-08T06:44:49.898Z{"msg":"acdba2ed-3906-411a-b598-7c403dee369a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117597291Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143042024-02-08T06:44:49.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117617139Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143052024-02-08T06:44:49.899Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117656076Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143062024-02-08T06:44:49.899Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117717962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143072024-02-08T06:44:49.899Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117766788Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143082024-02-08T06:44:49.899Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117841322Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143092024-02-08T06:44:49.899Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117888769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143102024-02-08T06:44:49.899Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117919656Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143112024-02-08T06:44:49.899Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.117966923Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143122024-02-08T06:44:49.899Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.118019249Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143132024-02-08T06:44:49.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.118047267Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143142024-02-08T06:44:49.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.118076915Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143152024-02-08T06:44:49.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.118092123Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143162024-02-08T06:44:49.899Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.118615784Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143172024-02-08T06:44:49.899Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.118644872Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143182024-02-08T06:44:49.899Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.118660461Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143192024-02-08T06:44:49.899Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.118696088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143202024-02-08T06:44:49.900Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.119158403Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143212024-02-08T06:44:49.900Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.119255096Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143222024-02-08T06:44:49.900Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.119711432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143232024-02-08T06:44:49.900Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.119750939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143242024-02-08T06:44:49.900Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.119767718Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143252024-02-08T06:44:49.900Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.119781537Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143262024-02-08T06:44:49.900Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.119803885Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143272024-02-08T06:44:49.900Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.119817844Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143282024-02-08T06:44:49.900Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.120021669Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143292024-02-08T06:44:49.900Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.120054486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143302024-02-08T06:44:49.905Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.120508432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143312024-02-08T06:44:49.905Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121050031Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143322024-02-08T06:44:49.905Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121098478Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143332024-02-08T06:44:49.905Z{"msg":"[0]R flush_numbers[0..12]: [219, 219, 217, 220, 216, 220, 209, 219, 219, 219, 220, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121136405Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143342024-02-08T06:44:49.905Z{"msg":"[0]R generation[0..12]: [130, 130, 128, 131, 128, 131, 125, 130, 130, 130, 131, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121173052Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143352024-02-08T06:44:49.905Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121209269Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143362024-02-08T06:44:49.906Z{"msg":"[1]R flush_numbers[0..12]: [219, 219, 217, 220, 216, 220, 209, 219, 219, 219, 220, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121244887Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143372024-02-08T06:44:49.906Z{"msg":"[1]R generation[0..12]: [130, 130, 128, 131, 128, 131, 125, 130, 130, 130, 131, 131]",Wait for a query_work_queue command to finish before sending IO
143382024-02-08T06:44:49.906Z"v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121302793Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143392024-02-08T06:44:49.906Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.12133751Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143402024-02-08T06:44:49.906Z{"msg":"[2]R flush_numbers[0..12]: [219, 219, 217, 220, 216, 220, 209, 219, 219, 219, 220, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121372317Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143412024-02-08T06:44:49.906Z{"msg":"[2]R generation[0..12]: [130, 130, 128, 131, 128, 131, 125, 130, 130, 130, 131, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121389446Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143422024-02-08T06:44:49.906Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121407994Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143432024-02-08T06:44:49.906Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121440992Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143442024-02-08T06:44:49.906Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121477219Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143452024-02-08T06:44:49.906Z{"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121504457Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143462024-02-08T06:44:49.906Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121540144Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143472024-02-08T06:44:49.906Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121576162Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143482024-02-08T06:44:49.906Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.12160031Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143492024-02-08T06:44:49.906Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121635907Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143502024-02-08T06:44:49.907Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121672555Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143512024-02-08T06:44:49.907Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121700832Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143522024-02-08T06:44:49.907Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.12173247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143532024-02-08T06:44:49.907Z{"msg":"acdba2ed-3906-411a-b598-7c403dee369a is now active with session: 88c7aec8-cacb-4b0d-94f4-2009c6625d77","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121767698Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143542024-02-08T06:44:49.907Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:49.121802855Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666,"session_id":"88c7aec8-cacb-4b0d-94f4-2009c6625d77"}
143552024-02-08T06:44:49.907Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
143562024-02-08T06:44:49.907Z,"time":"2024-02-08T06:44:49.121836102Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666}
143572024-02-08T06:44:49.907ZLoading write count information from file "/tmp/test_repair_verify.data"
143582024-02-08T06:44:49.907ZRun Repair workload
143592024-02-08T06:44:49.907Z01/30 Write block 274 len 32768 data: 8 8 9 11 9 13 11 10
143602024-02-08T06:44:49.907Z02/30 Flush
143612024-02-08T06:44:49.907Z03/30 Write block 375 len 12288 data: 8 10 9
143622024-02-08T06:44:49.907Z04/30 Write block 198 len 4096 data: 14
143632024-02-08T06:44:49.907Z05/30 Flush
143642024-02-08T06:44:49.911Z06/30 Write block 211 len 8192 data: 14 16
143652024-02-08T06:44:49.911Z07/30 Read block 572 len 16384
143662024-02-08T06:44:49.911Z08/30 Write block 74 len 24576 data: 10 9 10 7 8 9
143672024-02-08T06:44:49.911Z09/30 Write block 57 len 28672 data: 14 9 8 8 8 9 8
143682024-02-08T06:44:49.911Z10/30 Read block 332 len 36864
143692024-02-08T06:44:49.911Z11/30 Flush
143702024-02-08T06:44:49.914Z12/30 Write block 198 len 8192 data: 15 16
143712024-02-08T06:44:49.914Z13/30 Write block 44 len 24576 data: 6 4 7 5 6 5
143722024-02-08T06:44:49.914Z14/30 Write block 311 len 8192 data: 8 9
143732024-02-08T06:44:49.914Z15/30 Read block 1 len 12288
143742024-02-08T06:44:49.915Z16/30 Write block 298 len 40960 data: 8 11 13 15 16 13 13 15 15 11
143752024-02-08T06:44:49.918Z17/30 Read block 307 len 20480
143762024-02-08T06:44:49.918Z18/30 Write block 115 len 28672 data: 9 9 7 6 5 6 5
143772024-02-08T06:44:49.918Z19/30 Write block 585 len 28672 data: 11 10 10 8 11 11 11
143782024-02-08T06:44:49.921Z20/30 Write block 481 len 24576 data: 5 7 6 8 9 11
143792024-02-08T06:44:49.921Z21/30 Write block 81 len 36864 data: 9 10 9 9 9 9 10 10 9
143802024-02-08T06:44:49.921Z22/30 Write block 53 len 4096 data: 9
143812024-02-08T06:44:49.921Z23/30 Read block 307 len 4096
143822024-02-08T06:44:49.921Z24/30 Write block 97 len 8192 data: 17 15
143832024-02-08T06:44:49.924Z25/30 Write block 193 len 8192 data: 9 11
143842024-02-08T06:44:49.924Z26/30 Read block 456 len 20480
143852024-02-08T06:44:49.924Z27/30 Write block 35 len 32768 data: 9 8 7 9 7 7 7 6
143862024-02-08T06:44:49.924Z28/30 Write block 479 len 40960 data: 5 5 6 8 7 9 10 12 11 11
143872024-02-08T06:44:49.930Z29/30 Write block 430 len 36864 data: 10 9 10 7 10 11 13 15 13
143882024-02-08T06:44:49.930Z30/30 Read block 0 len 4096
143892024-02-08T06:44:49.930Z----------------------------------------------------------------
143902024-02-08T06:44:49.931Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
143912024-02-08T06:44:49.931ZStates: Active Active Active
143922024-02-08T06:44:49.931ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
143932024-02-08T06:44:49.931Z 1 Acked 1000 Write 8 Done Sent Done false
143942024-02-08T06:44:49.931Z 2 Acked 1001 Flush 0 Done Sent Done false
143952024-02-08T06:44:49.931Z 3 Acked 1002 Write 3 Done Sent Done false
143962024-02-08T06:44:49.931Z 4 Acked 1003 Write 1 Done Sent Done false
143972024-02-08T06:44:49.931Z 5 Acked 1004 Flush 0 Done Sent Done false
143982024-02-08T06:44:49.931Z 6 Acked 1005 Write 2 Done Sent Done false
143992024-02-08T06:44:49.931Z 7 Acked 1006 Read 4 Done Sent Done false
144002024-02-08T06:44:49.931Z 8 Acked 1007 Write 6 Done Sent Done false
144012024-02-08T06:44:49.931Z 9 Acked 1008 Write 7 Done Sent Done false
144022024-02-08T06:44:49.931Z 10 Acked 1009 Read 9 Done Sent Done false
144032024-02-08T06:44:49.931Z 11 Acked 1010 Flush 0 Done Sent Done false
144042024-02-08T06:44:49.931Z 12 Acked 1011 Write 2 Done Sent Done false
144052024-02-08T06:44:49.932Z 13 Acked 1012 Write 6 Done Sent Done false
144062024-02-08T06:44:49.932Z 14 Acked 1013 Write 2 Done Sent Done false
144072024-02-08T06:44:49.932Z 15 Acked 1014 Read 3 Done Sent Done false
144082024-02-08T06:44:49.932Z 16 Acked 1015 Write 10 Done Sent Done false
144092024-02-08T06:44:49.932Z 17 Acked 1016 Read 5 Done Sent Done false
144102024-02-08T06:44:49.932Z 18 Acked 1017 Write 7 Done Sent Done false
144112024-02-08T06:44:49.932Z 19 Acked 1018 Write 7 Done Sent Done false
144122024-02-08T06:44:49.932Z 20 Acked 1019 Write 6 Done Sent Done false
144132024-02-08T06:44:49.932Z 21 Acked 1020 Write 9 Done Sent Done false
144142024-02-08T06:44:49.932Z 22 Acked 1021 Write 1 Done Sent Done false
144152024-02-08T06:44:49.932Z 23 Acked 1022 Read 1 Done Sent Done false
144162024-02-08T06:44:49.932Z 24 Acked 1023 Write 2 Done Sent Done false
144172024-02-08T06:44:49.932Z 25 Acked 1024 Write 2 Done Sent Done false
144182024-02-08T06:44:49.932Z 26 Acked 1025 Read 5 Done Sent Done false
144192024-02-08T06:44:49.933Z 27 Acked 1026 Write 8 Done Sent Done false
144202024-02-08T06:44:49.933Z 28 Acked 1027 Write 10 Done Sent Done false
144212024-02-08T06:44:49.933Z 29 Acked 1028 Write 9 Done Sent Sent false
144222024-02-08T06:44:49.933Z 30 Acked 1029 Read 1 Done Sent Sent false
144232024-02-08T06:44:49.933Z STATES DS:0 DS:1 DS:2 TOTAL
144242024-02-08T06:44:49.933Z New 0 0 0 0
144252024-02-08T06:44:49.933Z Sent 0 30 2 32
144262024-02-08T06:44:49.933Z Done 30 0 28 58
144272024-02-08T06:44:49.933Z Skipped 0 0 0 0
144282024-02-08T06:44:49.933Z Error 0 0 0 0
144292024-02-08T06:44:49.933ZLast Flush: 1010 0 1010
144302024-02-08T06:44:49.933ZDownstairs last five completed:
144312024-02-08T06:44:49.933ZUpstairs last five completed: 30 29 28 27 26
144322024-02-08T06:44:49.933Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:49.151158581Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1666}
144332024-02-08T06:44:49.933ZWrote out file "/tmp/test_repair_verify.data"
144342024-02-08T06:44:49.937Z
144352024-02-08T06:44:49.937ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
144362024-02-08T06:44:49.953ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144372024-02-08T06:44:49.953Z 0 000-019 130 130 130 219 219 219 F F F
144382024-02-08T06:44:49.953Z 1 020-039 130 130 130 219 219 219 T F T <---
144392024-02-08T06:44:49.953Z 2 040-059 132 128 132 223 217 223 T F T <---
144402024-02-08T06:44:49.953Z 3 060-079 132 131 132 223 220 223 F F F <---
144412024-02-08T06:44:49.953Z 4 080-099 128 128 128 216 216 216 T F T <---
144422024-02-08T06:44:49.953Z 5 100-119 131 131 131 220 220 220 T F T <---
144432024-02-08T06:44:49.953Z 6 120-139 125 125 125 209 209 209 T F T <---
144442024-02-08T06:44:49.957Z 7 140-159 130 130 130 219 219 219 F F F
144452024-02-08T06:44:49.957Z 8 160-179 130 130 130 219 219 219 F F F
144462024-02-08T06:44:49.957Z 9 180-199 132 130 132 222 219 222 T F T <---
144472024-02-08T06:44:49.957Z 10 200-219 132 131 132 223 220 223 F F F <---
144482024-02-08T06:44:49.957Z 11 220-239 131 131 131 220 220 220 F F F
144492024-02-08T06:44:49.957Z 12 240-259 124 124 124 208 208 208 F F F
144502024-02-08T06:44:49.957Z 13 260-279 132 131 132 221 220 221 F F F <---
144512024-02-08T06:44:49.957Z 14 280-299 132 130 132 221 219 221 T F T <---
144522024-02-08T06:44:49.957Z 15 300-319 131 131 131 220 220 220 T F T <---
144532024-02-08T06:44:49.957Z 16 320-339 130 130 130 219 219 219 F F F
144542024-02-08T06:44:49.957Z 17 340-359 128 128 128 216 216 216 F F F
144552024-02-08T06:44:49.957Z 18 360-379 132 131 132 222 220 222 F F F <---
144562024-02-08T06:44:49.957Z 19 380-399 131 131 131 220 220 220 F F F
144572024-02-08T06:44:49.958Z 20 400-419 126 126 126 210 210 210 F F F
144582024-02-08T06:44:49.958Z 21 420-439 130 130 130 219 219 219 T F T <---
144592024-02-08T06:44:49.958Z 22 440-459 128 128 128 217 217 217 F F F
144602024-02-08T06:44:49.958Z 23 460-479 128 128 128 214 214 214 T F T <---
144612024-02-08T06:44:49.958Z 24 480-499 129 129 129 218 218 218 T F T <---
144622024-02-08T06:44:49.958Z 25 500-519 111 111 111 181 181 181 F F F
144632024-02-08T06:44:49.958Z 26 520-539 129 129 129 218 218 218 F F F
144642024-02-08T06:44:49.958Z 27 540-559 130 130 130 219 219 219 F F F
144652024-02-08T06:44:49.958Z 28 560-579 131 131 131 220 220 220 F F F
144662024-02-08T06:44:49.958Z 29 580-599 131 131 131 220 220 220 T F T <---
144672024-02-08T06:44:49.958ZMax gen: 132, Max flush: 223
144682024-02-08T06:44:49.958ZError: Difference in extent metadata found!
144692024-02-08T06:44:49.958ZA Difference in extent metadata is expected here
144702024-02-08T06:44:49.958ZOn loop 66
144712024-02-08T06:44:49.958Z
144722024-02-08T06:44:49.962ZVerifying data now
144732024-02-08T06:44:54.732ZNo change after verify
144742024-02-08T06:44:54.732ZLoop: 66 Downstairs dump after verify (and repair):
144752024-02-08T06:44:54.746ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144762024-02-08T06:44:54.747Z 0 000-019 130 130 130 219 219 219 F F F
144772024-02-08T06:44:54.747Z 1 020-039 133 133 133 224 224 224 F F F
144782024-02-08T06:44:54.747Z 2 040-059 133 133 133 224 224 224 F F F
144792024-02-08T06:44:54.747Z 3 060-079 132 132 132 223 223 223 F F F
144802024-02-08T06:44:54.747Z 4 080-099 133 133 133 224 224 224 F F F
144812024-02-08T06:44:54.747Z 5 100-119 133 133 133 224 224 224 F F F
144822024-02-08T06:44:54.747Z 6 120-139 133 133 133 224 224 224 F F F
144832024-02-08T06:44:54.750Z 7 140-159 130 130 130 219 219 219 F F F
144842024-02-08T06:44:54.750Z 8 160-179 130 130 130 219 219 219 F F F
144852024-02-08T06:44:54.750Z 9 180-199 133 133 133 224 224 224 F F F
144862024-02-08T06:44:54.750Z 10 200-219 132 132 132 223 223 223 F F F
144872024-02-08T06:44:54.750Z 11 220-239 131 131 131 220 220 220 F F F
144882024-02-08T06:44:54.750Z 12 240-259 124 124 124 208 208 208 F F F
144892024-02-08T06:44:54.750Z 13 260-279 132 132 132 221 221 221 F F F
144902024-02-08T06:44:54.750Z 14 280-299 133 133 133 224 224 224 F F F
144912024-02-08T06:44:54.751Z 15 300-319 133 133 133 224 224 224 F F F
144922024-02-08T06:44:54.751Z 16 320-339 130 130 130 219 219 219 F F F
144932024-02-08T06:44:54.751Z 17 340-359 128 128 128 216 216 216 F F F
144942024-02-08T06:44:54.751Z 18 360-379 132 132 132 222 222 222 F F F
144952024-02-08T06:44:54.751Z 19 380-399 131 131 131 220 220 220 F F F
144962024-02-08T06:44:54.751Z 20 400-419 126 126 126 210 210 210 F F F
144972024-02-08T06:44:54.751Z 21 420-439 133 133 133 224 224 224 F F F
144982024-02-08T06:44:54.751Z 22 440-459 128 128 128 217 217 217 F F F
144992024-02-08T06:44:54.751Z 23 460-479 133 133 133 224 224 224 F F F
145002024-02-08T06:44:54.751Z 24 480-499 133 133 133 224 224 224 F F F
145012024-02-08T06:44:54.751Z 25 500-519 111 111 111 181 181 181 F F F
145022024-02-08T06:44:54.751Z 26 520-539 129 129 129 218 218 218 F F F
145032024-02-08T06:44:54.751Z 27 540-559 130 130 130 219 219 219 F F F
145042024-02-08T06:44:54.751Z 28 560-579 131 131 131 220 220 220 F F F
145052024-02-08T06:44:54.751Z 29 580-599 133 133 133 224 224 224 F F F
145062024-02-08T06:44:54.752ZMax gen: 133, Max flush: 224
145072024-02-08T06:44:54.752Z
145082024-02-08T06:44:54.752ZBegin loop 67
145092024-02-08T06:44:54.752ZDownstairs to restart: 1
145102024-02-08T06:44:54.767Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.990650502Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674}
145112024-02-08T06:44:54.767Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\nCrucible runtime is spawned
145122024-02-08T06:44:54.767Z rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.990933201Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674}
145132024-02-08T06:44:54.772Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.990970118Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674}
145142024-02-08T06:44:54.772Z{"msg":"Crucible 4fb3cc70-73eb-4c5b-8fd1-e5be2b133905 has session id: 1ea90dde-9350-4acd-9559-ec749d165f2e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991004126Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145152024-02-08T06:44:54.772Z{"msg":"Upstairs opts: Upstairs UUID: 4fb3cc70-73eb-4c5b-8fd1-e5be2b133905, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991036933Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145162024-02-08T06:44:54.772Z{"msg":"Crucible stats registered with UUID: 4fb3cc70-73eb-4c5b-8fd1-e5be2b133905","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991072051Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145172024-02-08T06:44:54.773Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991096589Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674}
145182024-02-08T06:44:54.773Z{"msg":"4fb3cc70-73eb-4c5b-8fd1-e5be2b133905 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991157864Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145192024-02-08T06:44:54.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991195642Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145202024-02-08T06:44:54.773Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991236669Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145212024-02-08T06:44:54.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991265347Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145222024-02-08T06:44:54.773Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991305433Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145232024-02-08T06:44:54.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991333191Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145242024-02-08T06:44:54.773Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.99135816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145252024-02-08T06:44:54.773Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991419915Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145262024-02-08T06:44:54.773Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991451062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"io task","client":"2","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145272024-02-08T06:44:54.773Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.99148867Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145282024-02-08T06:44:54.773Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:53.991518807Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145292024-02-08T06:44:54.773Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:53.991562504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145302024-02-08T06:44:54.773Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991593402Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145312024-02-08T06:44:54.773Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991608441Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"io task","client":"2","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145322024-02-08T06:44:54.774Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:53.99162442Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145332024-02-08T06:44:54.774Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991653367Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145342024-02-08T06:44:54.774Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991669956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145352024-02-08T06:44:54.774Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.991744191Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145362024-02-08T06:44:54.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.992236534Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145372024-02-08T06:44:54.774Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.992263941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145382024-02-08T06:44:54.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.99227957Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145392024-02-08T06:44:54.774Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.992294779Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145402024-02-08T06:44:54.774Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.992865966Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145412024-02-08T06:44:54.774Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.992995857Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145422024-02-08T06:44:54.774Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.993521857Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145432024-02-08T06:44:54.774Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.993565914Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145442024-02-08T06:44:54.774Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.993584552Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145452024-02-08T06:44:54.774Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.99360904Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145462024-02-08T06:44:54.774Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.993626379Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145472024-02-08T06:44:54.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:53.993651127Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145482024-02-08T06:44:55.773Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.99324165Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145492024-02-08T06:44:55.789Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.993466823Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145502024-02-08T06:44:55.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.994208108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145512024-02-08T06:44:55.789Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.994238225Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145522024-02-08T06:44:55.789Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.994697351Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145532024-02-08T06:44:55.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995306855Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145542024-02-08T06:44:55.790Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995352482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145552024-02-08T06:44:55.790Z{"msg":"[0]R flush_numbers[0..12]: [219, 224, 224, 223, 224, 224, 224, 219, 219, 224, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.99537448Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145562024-02-08T06:44:55.790Z{"msg":"[0]R generation[0..12]: [130, 133, 133, 132, 133, 133, 133, 130, 130, 133, 132, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995402978Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145572024-02-08T06:44:55.790Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995421836Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145582024-02-08T06:44:55.790ZWait for a query_work_queue command to finish before sending IO
145592024-02-08T06:44:55.791Z{"msg":"[1]R flush_numbers[0..12]: [219, 224, 224, 223, 224, 224, 224, 219, 219, 224, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995456614Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145602024-02-08T06:44:55.791Z{"msg":"[1]R generation[0..12]: [130, 133, 133, 132, 133, 133, 133, 130, 130, 133, 132, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995482172Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145612024-02-08T06:44:55.791Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995500211Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145622024-02-08T06:44:55.791Z{"msg":"[2]R flush_numbers[0..12]: [219, 224, 224, 223, 224, 224, 224, 219, 219, 224, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995525969Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145632024-02-08T06:44:55.791Z{"msg":"[2]R generation[0..12]: [130, 133, 133, 132, 133, 133, 133, 130, 130, 133, 132, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995543557Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145642024-02-08T06:44:55.791Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995571065Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145652024-02-08T06:44:55.791Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995588294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145662024-02-08T06:44:55.792Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995618642Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145672024-02-08T06:44:55.792Z{"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995646189Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145682024-02-08T06:44:55.792Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995692206Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145692024-02-08T06:44:55.792Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995738633Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145702024-02-08T06:44:55.792Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.9957721Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145712024-02-08T06:44:55.792Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995808227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145722024-02-08T06:44:55.792Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995843915Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145732024-02-08T06:44:55.793Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995878702Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145742024-02-08T06:44:55.793Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995912869Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145752024-02-08T06:44:55.793Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
145762024-02-08T06:44:55.793Z4fb3cc70-73eb-4c5b-8fd1-e5be2b133905 is now active with session: 1ea90dde-9350-4acd-9559-ec749d165f2e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995951937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145772024-02-08T06:44:55.793Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995981104Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674,"session_id":"1ea90dde-9350-4acd-9559-ec749d165f2e"}
145782024-02-08T06:44:55.793Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:44:54.995994853Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674}
145792024-02-08T06:44:55.794ZLoading write count information from file "/tmp/test_repair_verify.data"
145802024-02-08T06:44:55.794ZRun Repair workload
145812024-02-08T06:44:55.794Z01/30 Write block 462 len 8192 data: 12 12
145822024-02-08T06:44:55.794Z02/30 Read block 375 len 24576
145832024-02-08T06:44:55.794Z03/30 Read block 556 len 24576
145842024-02-08T06:44:55.795Z04/30 Read block 132 len 12288
145852024-02-08T06:44:55.795Z05/30 Write block 276 len 28672 data: 10 12 10 14 12 11 9
145862024-02-08T06:44:55.795Z06/30 Read block 423 len 16384
145872024-02-08T06:44:55.802Z07/30 Read block 232 len 32768
145882024-02-08T06:44:55.802Z08/30 Read block 176 len 12288
145892024-02-08T06:44:55.803Z09/30 Read block 126 len 24576
145902024-02-08T06:44:55.803Z10/30 Read block 217 len 20480
145912024-02-08T06:44:55.804Z11/30 Write block 275 len 32768 data: 9 11 13 11 15 13 12 10
145922024-02-08T06:44:55.804Z12/30 Read block 374 len 16384
145932024-02-08T06:44:55.804Z13/30 Write block 291 len 28672 data: 14 13 11 9 7 7 6
145942024-02-08T06:44:55.805Z14/30 Write block 215 len 32768 data: 11 11 12 12 14 14 15 16
145952024-02-08T06:44:55.805Z15/30 Read block 289 len 40960
145962024-02-08T06:44:55.806Z16/30 Flush
145972024-02-08T06:44:55.806Z17/30 Write block 209 len 4096 data: 10
145982024-02-08T06:44:55.806Z18/30 Read block 190 len 4096
145992024-02-08T06:44:55.807Z19/30 Read block 475 len 16384
146002024-02-08T06:44:55.807Z20/30 Write block 360 len 32768 data: 9 6 7 6 10 10 8 11
146012024-02-08T06:44:55.807Z21/30 Flush
146022024-02-08T06:44:55.808Z22/30 Write block 507 len 28672 data: 6 6 8 10 7 6 7
146032024-02-08T06:44:55.808Z23/30 Read block 35 len 28672
146042024-02-08T06:44:55.808Z24/30 Write block 462 len 40960 data: 13 13 10 9 9 10 10 11 10 9
146052024-02-08T06:44:55.815Z25/30 Write block 35 len 16384 data: 10 9 8 10
146062024-02-08T06:44:55.815Z26/30 Write block 322 len 28672 data: 14 14 12 14 12 9 7
146072024-02-08T06:44:55.815Z27/30 Read block 529 len 20480
146082024-02-08T06:44:55.815Z28/30 Write block 10 len 24576 data: 10 8 8 8 7 6
146092024-02-08T06:44:55.816Z29/30 Read block 182 len 28672
146102024-02-08T06:44:55.816Z30/30 Read block 559 len 36864
146112024-02-08T06:44:55.816Z----------------------------------------------------------------
146122024-02-08T06:44:55.816Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
146132024-02-08T06:44:55.816ZStates: Active Active Active
146142024-02-08T06:44:55.816ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
146152024-02-08T06:44:55.817Z 1 Acked 1000 Write 2 Done Done Done false
146162024-02-08T06:44:55.817Z 2 Acked 1001 Read 6 Done Done Done false
146172024-02-08T06:44:55.817Z 3 Acked 1002 Read 6 Done Done Done false
146182024-02-08T06:44:55.817Z 4 Acked 1003 Read 3 Done Done Done false
146192024-02-08T06:44:55.817Z 5 Acked 1004 Write 7 Done Done Done false
146202024-02-08T06:44:55.818Z 6 Acked 1005 Read 4 Done Done Done false
146212024-02-08T06:44:55.818Z 7 Acked 1006 Read 8 Done Sent Done false
146222024-02-08T06:44:55.818Z 8 Acked 1007 Read 3 Done Sent Done false
146232024-02-08T06:44:55.818Z 9 Acked 1008 Read 6 Done Sent Done false
146242024-02-08T06:44:55.819Z 10 Acked 1009 Read 5 Done Sent Done false
146252024-02-08T06:44:55.819Z 11 Acked 1010 Write 8 Done Sent Done false
146262024-02-08T06:44:55.819Z 12 Acked 1011 Read 4 Done Sent Done false
146272024-02-08T06:44:55.819Z 13 Acked 1012 Write 7 Done Sent Done false
146282024-02-08T06:44:55.819Z 14 Acked 1013 Write 8 Done Sent Done false
146292024-02-08T06:44:55.820Z 15 Acked 1014 Read 10 Done Sent Done false
146302024-02-08T06:44:55.820Z 16 Acked 1015 Flush 0 Done Sent Done false
146312024-02-08T06:44:55.820Z 17 Acked 1016 Write 1 Done Sent Done false
146322024-02-08T06:44:55.820Z 18 Acked 1017 Read 1 Done Sent Done false
146332024-02-08T06:44:55.820Z 19 Acked 1018 Read 4 Done Sent Done false
146342024-02-08T06:44:55.820Z 20 Acked 1019 Write 8 Done Sent Done false
146352024-02-08T06:44:55.821Z 21 Acked 1020 Flush 0 Done Sent Done false
146362024-02-08T06:44:55.821Z 22 Acked 1021 Write 7 Done Sent Done false
146372024-02-08T06:44:55.821Z 23 Acked 1022 Read 7 Done Sent Done false
146382024-02-08T06:44:55.821Z 24 Acked 1023 Write 10 Done Sent Done false
146392024-02-08T06:44:55.821Z 25 Acked 1024 Write 4 Done Sent Done false
146402024-02-08T06:44:55.821Z 26 Acked 1025 Write 7 Done Sent Done false
146412024-02-08T06:44:55.822Z 27 Acked 1026 Read 5 Done Sent Done false
146422024-02-08T06:44:55.822Z 28 Acked 1027 Write 6 Done Sent Done false
146432024-02-08T06:44:55.822Z 29 Acked 1028 Read 7 Done Sent Done false
146442024-02-08T06:44:55.822Z 30 Acked 1029 Read 9 Done Sent Done false
146452024-02-08T06:44:55.822Z STATES DS:0 DS:1 DS:2 TOTAL
146462024-02-08T06:44:55.823Z New 0 0 0 0
146472024-02-08T06:44:55.823Z Sent 0 24 0 24
146482024-02-08T06:44:55.823Z Done 30 6 30 66
146492024-02-08T06:44:55.823Z Skipped 0 0 0 0
146502024-02-08T06:44:55.824Z Error 0 0 0 0
146512024-02-08T06:44:55.824ZLast Flush: 1020 0 1020
146522024-02-08T06:44:55.824ZDownstairs last five completed:
146532024-02-08T06:44:55.824ZUpstairs last five completed: 30 29 28 27 26
146542024-02-08T06:44:55.825Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:44:55.029224149Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1674}
146552024-02-08T06:44:55.825ZWrote out file "/tmp/test_repair_verify.data"
146562024-02-08T06:44:55.825Z
146572024-02-08T06:44:55.831ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
146582024-02-08T06:44:55.838ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146592024-02-08T06:44:55.838Z 0 000-019 130 130 130 219 219 219 T F T <---
146602024-02-08T06:44:55.838Z 1 020-039 133 133 133 224 224 224 T F T <---
146612024-02-08T06:44:55.838Z 2 040-059 133 133 133 224 224 224 F F F
146622024-02-08T06:44:55.838Z 3 060-079 132 132 132 223 223 223 F F F
146632024-02-08T06:44:55.839Z 4 080-099 133 133 133 224 224 224 F F F
146642024-02-08T06:44:55.839Z 5 100-119 133 133 133 224 224 224 F F F
146652024-02-08T06:44:55.839Z 6 120-139 133 133 133 224 224 224 F F F
146662024-02-08T06:44:55.839Z 7 140-159 130 130 130 219 219 219 F F F
146672024-02-08T06:44:55.839Z 8 160-179 130 130 130 219 219 219 F F F
146682024-02-08T06:44:55.840Z 9 180-199 133 133 133 224 224 224 F F F
146692024-02-08T06:44:55.840Z 10 200-219 134 132 134 226 223 226 F F F <---
146702024-02-08T06:44:55.840Z 11 220-239 134 131 134 225 220 225 F F F <---
146712024-02-08T06:44:55.840Z 12 240-259 124 124 124 208 208 208 F F F
146722024-02-08T06:44:55.841Z 13 260-279 134 132 134 225 221 225 F T F <---
146732024-02-08T06:44:55.841Z 14 280-299 134 133 134 225 224 225 F T F <---
146742024-02-08T06:44:55.841Z 15 300-319 133 133 133 224 224 224 F F F
146752024-02-08T06:44:55.841Z 16 320-339 130 130 130 219 219 219 T F T <---
146762024-02-08T06:44:55.841Z 17 340-359 128 128 128 216 216 216 F F F
146772024-02-08T06:44:55.842Z 18 360-379 134 132 134 226 222 226 F F F <---
146782024-02-08T06:44:55.842Z 19 380-399 131 131 131 220 220 220 F F F
146792024-02-08T06:44:55.842Z 20 400-419 126 126 126 210 210 210 F F F
146802024-02-08T06:44:55.842Z 21 420-439 133 133 133 224 224 224 F F F
146812024-02-08T06:44:55.842Z 22 440-459 128 128 128 217 217 217 F F F
146822024-02-08T06:44:55.843Z 23 460-479 134 133 134 225 224 225 T T T <---
146832024-02-08T06:44:55.843Z 24 480-499 133 133 133 224 224 224 F F F
146842024-02-08T06:44:55.843Z 25 500-519 111 111 111 181 181 181 T F T <---
146852024-02-08T06:44:55.843Z 26 520-539 129 129 129 218 218 218 F F F
146862024-02-08T06:44:55.843Z 27 540-559 130 130 130 219 219 219 F F F
146872024-02-08T06:44:55.844Z 28 560-579 131 131 131 220 220 220 F F F
146882024-02-08T06:44:55.844Z 29 580-599 133 133 133 224 224 224 F F F
146892024-02-08T06:44:55.844ZMax gen: 134, Max flush: 226
146902024-02-08T06:44:55.844ZError: Difference in extent metadata found!
146912024-02-08T06:44:55.844ZA Difference in extent metadata is expected here
146922024-02-08T06:44:55.845ZOn loop 67
146932024-02-08T06:44:55.845Z
146942024-02-08T06:44:55.845ZVerifying data now
146952024-02-08T06:45:01.357ZNo change after verify
146962024-02-08T06:45:01.357ZLoop: 67 Downstairs dump after verify (and repair):
146972024-02-08T06:45:01.372ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146982024-02-08T06:45:01.372Z 0 000-019 135 135 135 227 227 227 F F F
146992024-02-08T06:45:01.372Z 1 020-039 135 135 135 227 227 227 F F F
147002024-02-08T06:45:01.373Z 2 040-059 133 133 133 224 224 224 F F F
147012024-02-08T06:45:01.373Z 3 060-079 132 132 132 223 223 223 F F F
147022024-02-08T06:45:01.373Z 4 080-099 133 133 133 224 224 224 F F F
147032024-02-08T06:45:01.373Z 5 100-119 133 133 133 224 224 224 F F F
147042024-02-08T06:45:01.373Z 6 120-139 133 133 133 224 224 224 F F F
147052024-02-08T06:45:01.376Z 7 140-159 130 130 130 219 219 219 F F F
147062024-02-08T06:45:01.376Z 8 160-179 130 130 130 219 219 219 F F F
147072024-02-08T06:45:01.376Z 9 180-199 133 133 133 224 224 224 F F F
147082024-02-08T06:45:01.377Z 10 200-219 134 134 134 226 226 226 F F F
147092024-02-08T06:45:01.377Z 11 220-239 134 134 134 225 225 225 F F F
147102024-02-08T06:45:01.377Z 12 240-259 124 124 124 208 208 208 F F F
147112024-02-08T06:45:01.377Z 13 260-279 134 134 134 225 225 225 F F F
147122024-02-08T06:45:01.377Z 14 280-299 134 134 134 225 225 225 F F F
147132024-02-08T06:45:01.377Z 15 300-319 133 133 133 224 224 224 F F F
147142024-02-08T06:45:01.377Z 16 320-339 135 135 135 227 227 227 F F F
147152024-02-08T06:45:01.377Z 17 340-359 128 128 128 216 216 216 F F F
147162024-02-08T06:45:01.377Z 18 360-379 134 134 134 226 226 226 F F F
147172024-02-08T06:45:01.377Z 19 380-399 131 131 131 220 220 220 F F F
147182024-02-08T06:45:01.377Z 20 400-419 126 126 126 210 210 210 F F F
147192024-02-08T06:45:01.377Z 21 420-439 133 133 133 224 224 224 F F F
147202024-02-08T06:45:01.377Z 22 440-459 128 128 128 217 217 217 F F F
147212024-02-08T06:45:01.377Z 23 460-479 135 135 135 227 227 227 F F F
147222024-02-08T06:45:01.377Z 24 480-499 133 133 133 224 224 224 F F F
147232024-02-08T06:45:01.378Z 25 500-519 135 135 135 227 227 227 F F F
147242024-02-08T06:45:01.378Z 26 520-539 129 129 129 218 218 218 F F F
147252024-02-08T06:45:01.378Z 27 540-559 130 130 130 219 219 219 F F F
147262024-02-08T06:45:01.378Z 28 560-579 131 131 131 220 220 220 F F F
147272024-02-08T06:45:01.378Z 29 580-599 133 133 133 224 224 224 F F F
147282024-02-08T06:45:01.378ZMax gen: 135, Max flush: 227
147292024-02-08T06:45:01.378Z
147302024-02-08T06:45:01.378ZBegin loop 68
147312024-02-08T06:45:01.378ZDownstairs to restart: 2
147322024-02-08T06:45:01.393Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616250326Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682}
147332024-02-08T06:45:01.394Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616486579Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682}
147342024-02-08T06:45:01.394Z{Crucible runtime is spawned
147352024-02-08T06:45:01.400Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616543434Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682}
147362024-02-08T06:45:01.400Z{"msg":"Crucible 7b3409bf-7943-4763-a062-e5b142f3c3ef has session id: fc136117-c4f6-442a-b3b5-d49d66de73a6","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616572042Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147372024-02-08T06:45:01.400Z{"msg":"Upstairs opts: Upstairs UUID: 7b3409bf-7943-4763-a062-e5b142f3c3ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.61660887Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147382024-02-08T06:45:01.400Z{"msg":"Crucible stats registered with UUID: 7b3409bf-7943-4763-a062-e5b142f3c3ef","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616628928Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147392024-02-08T06:45:01.400Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616650746Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682}
147402024-02-08T06:45:01.400Z{"msg":"7b3409bf-7943-4763-a062-e5b142f3c3ef active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.61673848Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147412024-02-08T06:45:01.400Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616765988Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147422024-02-08T06:45:01.400Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616798335Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147432024-02-08T06:45:01.400Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616843362Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147442024-02-08T06:45:01.400Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.61687345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147452024-02-08T06:45:01.400Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616910247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147462024-02-08T06:45:01.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616944804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147472024-02-08T06:45:01.401Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.616978652Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"io task","client":"0","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147482024-02-08T06:45:01.401Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.617013859Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147492024-02-08T06:45:01.401Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.617040777Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147502024-02-08T06:45:01.401Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.617067345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"io task","client":"0","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147512024-02-08T06:45:01.401Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.617093123Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147522024-02-08T06:45:01.401Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:00.617184396Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147532024-02-08T06:45:01.401Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:00.617230423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147542024-02-08T06:45:01.401Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:00.617341055Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147552024-02-08T06:45:01.401Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.617385611Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147562024-02-08T06:45:01.401Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.617450156Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147572024-02-08T06:45:01.401Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.617479844Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147582024-02-08T06:45:01.401Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.617924881Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147592024-02-08T06:45:01.401Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.617962918Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147602024-02-08T06:45:01.401Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.617979336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147612024-02-08T06:45:01.401Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.618015204Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147622024-02-08T06:45:01.402Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.618530845Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147632024-02-08T06:45:01.402Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.618641027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147642024-02-08T06:45:01.402Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.619038447Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147652024-02-08T06:45:01.402Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.619076924Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147662024-02-08T06:45:01.402Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.619104032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147672024-02-08T06:45:01.402Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.619198115Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147682024-02-08T06:45:01.402Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.619242922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147692024-02-08T06:45:01.402Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:00.619277639Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147702024-02-08T06:45:02.397Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.61916554Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147712024-02-08T06:45:02.403Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.619393932Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147722024-02-08T06:45:02.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.620443313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147732024-02-08T06:45:02.403Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.620476831Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147742024-02-08T06:45:02.403Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.620955385Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147752024-02-08T06:45:02.403Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621456317Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147762024-02-08T06:45:02.403Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621488175Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147772024-02-08T06:45:02.404Z{"msg":"[0]R flush_numbers[0..12]: [227, 227, 224, 223, 224, 224, 224, 219, 219, 224, 226, 225]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621503644Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147782024-02-08T06:45:02.404Z{"msg":"[0]R generation[0..12]: [135, 135, 133, 132, 133, 133, 133, 130, 130, 133, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621525672Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147792024-02-08T06:45:02.404Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621540961Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147802024-02-08T06:45:02.404Z{"msg":"[1]R flush_numbers[0..12]: [227, 227, 224, 223, 224, 224, 224, 219, 219, 224, 226, 225]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621562829Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147812024-02-08T06:45:02.404Z{"msg":"[1]R generation[0..12]: [135, 135, 133, 132, 133, 133, 133, 130, 130, 133, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621577898Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147822024-02-08T06:45:02.404Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621598827Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147832024-02-08T06:45:02.404Z{"msg":"[2]R flush_numbers[0..12]: [227, 227, 224, 223, 224, 224, 224, 219, 219, 224, 226, 225]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621613376Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147842024-02-08T06:45:02.404Z{"msg":"[2]R generation[0..12]: [135, 135, 133, 132, 133, 133, 133, 130, 130, 133, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621634324Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147852024-02-08T06:45:02.404Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
147862024-02-08T06:45:02.404Ztime":"2024-02-08T06:45:01.621648743Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147872024-02-08T06:45:02.404Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621665762Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147882024-02-08T06:45:02.404Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621679941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147892024-02-08T06:45:02.404Z{"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621700339Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147902024-02-08T06:45:02.404Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621718538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147912024-02-08T06:45:02.404Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621743486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147922024-02-08T06:45:02.405Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621771224Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147932024-02-08T06:45:02.405Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621795752Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147942024-02-08T06:45:02.405Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621828569Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147952024-02-08T06:45:02.405Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621855158Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147962024-02-08T06:45:02.405Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621871926Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147972024-02-08T06:45:02.405Z{"msg":"7b3409bf-7943-4763-a062-e5b142f3c3ef is now active with session: fc136117-c4f6-442a-b3b5-d49d66de73a6","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621903244Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147982024-02-08T06:45:02.405Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.621934402Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682,"session_id":"fc136117-c4f6-442a-b3b5-d49d66de73a6"}
147992024-02-08T06:45:02.405Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:01.6219552Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682}
148002024-02-08T06:45:02.405ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148012024-02-08T06:45:02.405ZLoading write count information from file "/tmp/test_repair_verify.data"
148022024-02-08T06:45:02.405ZRun Repair workload
148032024-02-08T06:45:02.405Z01/30 Write block 255 len 32768 data: 12 10 8 8 11 11 15 15
148042024-02-08T06:45:02.409Z02/30 Write block 520 len 40960 data: 10 8 8 9 10 10 10 9 11 11
148052024-02-08T06:45:02.409Z03/30 Write block 564 len 20480 data: 11 12 11 11 12
148062024-02-08T06:45:02.409Z04/30 Read block 250 len 16384
148072024-02-08T06:45:02.409Z05/30 Read block 314 len 20480
148082024-02-08T06:45:02.409Z06/30 Read block 340 len 32768
148092024-02-08T06:45:02.409Z07/30 Read block 466 len 32768
148102024-02-08T06:45:02.409Z08/30 Read block 309 len 40960
148112024-02-08T06:45:02.409Z09/30 Read block 573 len 24576
148122024-02-08T06:45:02.415Z10/30 Flush
148132024-02-08T06:45:02.416Z11/30 Write block 398 len 28672 data: 8 9 9 11 11 10 11
148142024-02-08T06:45:02.416Z12/30 Read block 151 len 20480
148152024-02-08T06:45:02.419Z13/30 Read block 125 len 32768
148162024-02-08T06:45:02.419Z14/30 Write block 241 len 4096 data: 10
148172024-02-08T06:45:02.419Z15/30 Read block 527 len 20480
148182024-02-08T06:45:02.420Z16/30 Read block 37 len 32768
148192024-02-08T06:45:02.420Z17/30 Read block 541 len 36864
148202024-02-08T06:45:02.420Z18/30 Read block 291 len 8192
148212024-02-08T06:45:02.420Z19/30 Write block 408 len 20480 data: 10 11 9 8 9
148222024-02-08T06:45:02.420Z20/30 Read block 474 len 28672
148232024-02-08T06:45:02.423Z21/30 Read block 231 len 28672
148242024-02-08T06:45:02.423Z22/30 Flush
148252024-02-08T06:45:02.427Z23/30 Write block 451 len 20480 data: 10 11 12 12 15
148262024-02-08T06:45:02.427Z24/30 Read block 427 len 32768
148272024-02-08T06:45:02.427Z25/30 Read block 220 len 28672
148282024-02-08T06:45:02.427Z26/30 Flush
148292024-02-08T06:45:02.430Z27/30 Write block 372 len 8192 data: 14 13
148302024-02-08T06:45:02.430Z28/30 Read block 547 len 28672
148312024-02-08T06:45:02.430Z29/30 Write block 92 len 20480 data: 16 15 17 17 17
148322024-02-08T06:45:02.431Z30/30 Read block 286 len 28672
148332024-02-08T06:45:02.435Z----------------------------------------------------------------
148342024-02-08T06:45:02.435Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30
148352024-02-08T06:45:02.435ZStates: Active Active Active
148362024-02-08T06:45:02.435ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
148372024-02-08T06:45:02.435Z 1 Acked 1000 Write 8 Done Done Done false
148382024-02-08T06:45:02.435Z 2 Acked 1001 Write 10 Done Done Done false
148392024-02-08T06:45:02.435Z 3 Acked 1002 Write 5 Done Done Done false
148402024-02-08T06:45:02.435Z 4 Acked 1003 Read 4 Done Done Done false
148412024-02-08T06:45:02.435Z 5 Acked 1004 Read 5 Done Done Done false
148422024-02-08T06:45:02.435Z 6 Acked 1005 Read 8 Done Done Sent false
148432024-02-08T06:45:02.435Z 7 Acked 1006 Read 8 Done Done Sent false
148442024-02-08T06:45:02.435Z 8 Acked 1007 Read 10 Done Done Sent false
148452024-02-08T06:45:02.435Z 9 Acked 1008 Read 6 Done Done Sent false
148462024-02-08T06:45:02.436Z 10 Acked 1009 Flush 0 Done Done Sent false
148472024-02-08T06:45:02.436Z 11 Acked 1010 Write 7 Done Done Sent false
148482024-02-08T06:45:02.436Z 12 Acked 1011 Read 5 Done Done Sent false
148492024-02-08T06:45:02.436Z 13 Acked 1012 Read 8 Done Done Sent false
148502024-02-08T06:45:02.436Z 14 Acked 1013 Write 1 Done Done Sent false
148512024-02-08T06:45:02.436Z 15 Acked 1014 Read 5 Done Done Sent false
148522024-02-08T06:45:02.436Z 16 Acked 1015 Read 8 Done Done Sent false
148532024-02-08T06:45:02.436Z 17 Acked 1016 Read 9 Done Done Sent false
148542024-02-08T06:45:02.436Z 18 Acked 1017 Read 2 Done Done Sent false
148552024-02-08T06:45:02.436Z 19 Acked 1018 Write 5 Done Done Sent false
148562024-02-08T06:45:02.436Z 20 Acked 1019 Read 7 Done Done Sent false
148572024-02-08T06:45:02.436Z 21 Acked 1020 Read 7 Done Done Sent false
148582024-02-08T06:45:02.436Z 22 Acked 1021 Flush 0 Done Done Sent false
148592024-02-08T06:45:02.436Z 23 Acked 1022 Write 5 Done Done Sent false
148602024-02-08T06:45:02.436Z 24 Acked 1023 Read 8 Done Done Sent false
148612024-02-08T06:45:02.437Z 25 Acked 1024 Read 7 Done Done Sent false
148622024-02-08T06:45:02.437Z 26 Acked 1025 Flush 0 Done Done Sent false
148632024-02-08T06:45:02.437Z 27 Acked 1026 Write 2 Done Done Sent false
148642024-02-08T06:45:02.437Z 28 Acked 1027 Read 7 Done Done Sent false
148652024-02-08T06:45:02.437Z 29 Acked 1028 Write 5 Done Done Sent false
148662024-02-08T06:45:02.437Z 30 Acked 1029 Read 7 Sent Done Sent false
148672024-02-08T06:45:02.437Z STATES DS:0 DS:1 DS:2 TOTAL
148682024-02-08T06:45:02.437Z New 0 0 0 0
148692024-02-08T06:45:02.437Z Sent 1 0 25 26
148702024-02-08T06:45:02.437Z Done 29 30 5 64
148712024-02-08T06:45:02.437Z Skipped 0 0 0 0
148722024-02-08T06:45:02.437Z Error 0 0 0 0
148732024-02-08T06:45:02.437ZLast Flush: 1025 1025 0
148742024-02-08T06:45:02.437ZDownstairs last five completed:
148752024-02-08T06:45:02.437ZUpstairs last five completed: 30 29 28 27 26
148762024-02-08T06:45:02.437Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:01.654978071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1682}
148772024-02-08T06:45:02.438ZWrote out file "/tmp/test_repair_verify.data"
148782024-02-08T06:45:02.442Z
148792024-02-08T06:45:02.442ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
148802024-02-08T06:45:02.458ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148812024-02-08T06:45:02.458Z 0 000-019 135 135 135 227 227 227 F F F
148822024-02-08T06:45:02.458Z 1 020-039 135 135 135 227 227 227 F F F
148832024-02-08T06:45:02.458Z 2 040-059 133 133 133 224 224 224 F F F
148842024-02-08T06:45:02.459Z 3 060-079 132 132 132 223 223 223 F F F
148852024-02-08T06:45:02.459Z 4 080-099 133 133 133 224 224 224 T T F <---
148862024-02-08T06:45:02.471Z 5 100-119 133 133 133 224 224 224 F F F
148872024-02-08T06:45:02.471Z 6 120-139 133 133 133 224 224 224 F F F
148882024-02-08T06:45:02.471Z 7 140-159 130 130 130 219 219 219 F F F
148892024-02-08T06:45:02.471Z 8 160-179 130 130 130 219 219 219 F F F
148902024-02-08T06:45:02.471Z 9 180-199 133 133 133 224 224 224 F F F
148912024-02-08T06:45:02.471Z 10 200-219 134 134 134 226 226 226 F F F
148922024-02-08T06:45:02.471Z 11 220-239 134 134 134 225 225 225 F F F
148932024-02-08T06:45:02.471Z 12 240-259 136 136 124 229 229 208 F F T <---
148942024-02-08T06:45:02.471Z 13 260-279 136 136 134 228 228 225 F F T <---
148952024-02-08T06:45:02.471Z 14 280-299 134 134 134 225 225 225 F F F
148962024-02-08T06:45:02.472Z 15 300-319 133 133 133 224 224 224 F F F
148972024-02-08T06:45:02.472Z 16 320-339 135 135 135 227 227 227 F F F
148982024-02-08T06:45:02.472Z 17 340-359 128 128 128 216 216 216 F F F
148992024-02-08T06:45:02.472Z 18 360-379 134 134 134 226 226 226 T T F <---
149002024-02-08T06:45:02.472Z 19 380-399 136 136 131 229 229 220 F F F <---
149012024-02-08T06:45:02.472Z 20 400-419 136 136 126 229 229 210 F F F <---
149022024-02-08T06:45:02.472Z 21 420-439 133 133 133 224 224 224 F F F
149032024-02-08T06:45:02.472Z 22 440-459 136 136 128 230 230 217 F F F <---
149042024-02-08T06:45:02.472Z 23 460-479 135 135 135 227 227 227 F F F
149052024-02-08T06:45:02.472Z 24 480-499 133 133 133 224 224 224 F F F
149062024-02-08T06:45:02.472Z 25 500-519 135 135 135 227 227 227 F F F
149072024-02-08T06:45:02.472Z 26 520-539 136 136 129 228 228 218 F F T <---
149082024-02-08T06:45:02.472Z 27 540-559 130 130 130 219 219 219 F F F
149092024-02-08T06:45:02.472Z 28 560-579 136 136 131 228 228 220 F F T <---
149102024-02-08T06:45:02.472Z 29 580-599 133 133 133 224 224 224 F F F
149112024-02-08T06:45:02.473ZMax gen: 136, Max flush: 230
149122024-02-08T06:45:02.473ZError: Difference in extent metadata found!
149132024-02-08T06:45:02.473ZA Difference in extent metadata is expected here
149142024-02-08T06:45:02.473ZOn loop 68
149152024-02-08T06:45:02.473Z
149162024-02-08T06:45:02.473ZVerifying data now
149172024-02-08T06:45:07.945ZNo change after verify
149182024-02-08T06:45:07.945ZLoop: 68 Downstairs dump after verify (and repair):
149192024-02-08T06:45:07.967ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149202024-02-08T06:45:07.967Z 0 000-019 135 135 135 227 227 227 F F F
149212024-02-08T06:45:07.967Z 1 020-039 135 135 135 227 227 227 F F F
149222024-02-08T06:45:07.967Z 2 040-059 133 133 133 224 224 224 F F F
149232024-02-08T06:45:07.967Z 3 060-079 132 132 132 223 223 223 F F F
149242024-02-08T06:45:07.968Z 4 080-099 137 137 137 231 231 231 F F F
149252024-02-08T06:45:07.968Z 5 100-119 133 133 133 224 224 224 F F F
149262024-02-08T06:45:07.968Z 6 120-139 133 133 133 224 224 224 F F F
149272024-02-08T06:45:07.968Z 7 140-159 130 130 130 219 219 219 F F F
149282024-02-08T06:45:07.968Z 8 160-179 130 130 130 219 219 219 F F F
149292024-02-08T06:45:07.968Z 9 180-199 133 133 133 224 224 224 F F F
149302024-02-08T06:45:07.968Z 10 200-219 134 134 134 226 226 226 F F F
149312024-02-08T06:45:07.969Z 11 220-239 134 134 134 225 225 225 F F F
149322024-02-08T06:45:07.969Z 12 240-259 136 136 136 229 229 229 F F F
149332024-02-08T06:45:07.969Z 13 260-279 136 136 136 228 228 228 F F F
149342024-02-08T06:45:07.969Z 14 280-299 134 134 134 225 225 225 F F F
149352024-02-08T06:45:07.969Z 15 300-319 133 133 133 224 224 224 F F F
149362024-02-08T06:45:07.969Z 16 320-339 135 135 135 227 227 227 F F F
149372024-02-08T06:45:07.969Z 17 340-359 128 128 128 216 216 216 F F F
149382024-02-08T06:45:07.969Z 18 360-379 137 137 137 231 231 231 F F F
149392024-02-08T06:45:07.969Z 19 380-399 136 136 136 229 229 229 F F F
149402024-02-08T06:45:07.969Z 20 400-419 136 136 136 229 229 229 F F F
149412024-02-08T06:45:07.969Z 21 420-439 133 133 133 224 224 224 F F F
149422024-02-08T06:45:07.969Z 22 440-459 136 136 136 230 230 230 F F F
149432024-02-08T06:45:07.970Z 23 460-479 135 135 135 227 227 227 F F F
149442024-02-08T06:45:07.970Z 24 480-499 133 133 133 224 224 224 F F F
149452024-02-08T06:45:07.970Z 25 500-519 135 135 135 227 227 227 F F F
149462024-02-08T06:45:07.970Z 26 520-539 136 136 136 228 228 228 F F F
149472024-02-08T06:45:07.970Z 27 540-559 130 130 130 219 219 219 F F F
149482024-02-08T06:45:07.970Z 28 560-579 136 136 136 228 228 228 F F F
149492024-02-08T06:45:07.970Z 29 580-599 133 133 133 224 224 224 F F F
149502024-02-08T06:45:07.970ZMax gen: 137, Max flush: 231
149512024-02-08T06:45:07.974Z
149522024-02-08T06:45:07.974ZBegin loop 69
149532024-02-08T06:45:07.974ZDownstairs to restart: 0
149542024-02-08T06:45:07.979Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.198653233Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690}
149552024-02-08T06:45:07.979Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.198985368Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690}
149562024-02-08T06:45:07.979Z{"Crucible runtime is spawned
149572024-02-08T06:45:07.979Zmsg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199043364Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690}
149582024-02-08T06:45:07.979Z{"msg":"Crucible 82e72914-cfa8-498f-b3d2-a5922386fa3e has session id: 214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199071062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149592024-02-08T06:45:07.979Z{"msg":"Upstairs opts: Upstairs UUID: 82e72914-cfa8-498f-b3d2-a5922386fa3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.19909886Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149602024-02-08T06:45:07.979Z{"msg":"Crucible stats registered with UUID: 82e72914-cfa8-498f-b3d2-a5922386fa3e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199129187Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149612024-02-08T06:45:07.979Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199143336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690}
149622024-02-08T06:45:07.979Z{"msg":"82e72914-cfa8-498f-b3d2-a5922386fa3e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.1992305Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149632024-02-08T06:45:07.979Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199256478Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149642024-02-08T06:45:07.979Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199288816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149652024-02-08T06:45:07.979Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199306984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149662024-02-08T06:45:07.979Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199327673Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149672024-02-08T06:45:07.979Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199342291Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149682024-02-08T06:45:07.980Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.19936176Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149692024-02-08T06:45:07.980Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199396967Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149702024-02-08T06:45:07.980Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199425265Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149712024-02-08T06:45:07.980Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199538257Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"io task","client":"2","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149722024-02-08T06:45:07.980Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199587023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149732024-02-08T06:45:07.980Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199617141Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149742024-02-08T06:45:07.980Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.199650648Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"io task","client":"2","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149752024-02-08T06:45:07.980Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.20015294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149762024-02-08T06:45:07.980Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.200214906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149772024-02-08T06:45:07.986Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.200269182Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149782024-02-08T06:45:07.986Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.200331097Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149792024-02-08T06:45:07.986Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.200708079Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149802024-02-08T06:45:07.986Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.20082515Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149812024-02-08T06:45:07.986Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.201326543Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149822024-02-08T06:45:07.987Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.20136586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149832024-02-08T06:45:07.987Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.201381828Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149842024-02-08T06:45:07.987Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.201452063Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149852024-02-08T06:45:07.987Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.201468392Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149862024-02-08T06:45:07.987Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.201507919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149872024-02-08T06:45:07.987Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.201531947Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149882024-02-08T06:45:07.987Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.201563065Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149892024-02-08T06:45:07.987Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.201983713Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149902024-02-08T06:45:07.987Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.20242014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149912024-02-08T06:45:07.987Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202447478Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149922024-02-08T06:45:07.987Z{"msg":"[0]R flush_numbers[0..12]: [227, 227, 224, 223, 231, 224, 224, 219, 219, 224, 226, 225]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202463127Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149932024-02-08T06:45:07.987Z{"msg":"[0]R generation[0..12]: [135, 135, 133, 132, 137, 133, 133, 130, 130, 133, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202477986Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149942024-02-08T06:45:07.987Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202500614Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149952024-02-08T06:45:07.987Z{"msg":"[1]R flush_numbers[0..12]: [227, 227, 224, 223, 231, 224, 224, 219, 219, 224, 226, 225]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202515353Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149962024-02-08T06:45:07.987Z{"msg":"[1]R generation[0..12]: [135, 135, 133, 132, 137, 133, 133, 130, 130, 133, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202536062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149972024-02-08T06:45:07.988Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.20255048Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149982024-02-08T06:45:07.988Z{"msg":"[2]R flush_numbers[0..12]: [227, 227, 224, 223, 231, 224, 224, 219, 219, 224, 226, 225]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202572249Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
149992024-02-08T06:45:07.988Z{"msg":"[2]R generation[0..12]: [135, 135, 133, 132, 137, 133, 133, 130, 130, 133, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202586298Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150002024-02-08T06:45:07.988Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202606626Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150012024-02-08T06:45:07.988Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202620355Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150022024-02-08T06:45:07.988ZWait for a query_work_queue command to finish before sending IO
150032024-02-08T06:45:07.988Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202644664Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150042024-02-08T06:45:07.988Z{"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202658982Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150052024-02-08T06:45:07.988Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202679101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150062024-02-08T06:45:07.988Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.20269279Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150072024-02-08T06:45:07.988Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202711339Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150082024-02-08T06:45:07.988Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202724918Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150092024-02-08T06:45:07.988Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202744586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150102024-02-08T06:45:07.988Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202758155Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150112024-02-08T06:45:07.988Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202776703Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150122024-02-08T06:45:07.989Z{"msg":"82e72914-cfa8-498f-b3d2-a5922386fa3e is now active with session: 214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202789793Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150132024-02-08T06:45:07.989Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.202808181Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690,"session_id":"214c3ab1-e99d-4f3c-ac00-6c3a1a6e0e05"}
150142024-02-08T06:45:07.989Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:07.20282091Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690}
150152024-02-08T06:45:07.989ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150162024-02-08T06:45:07.989ZLoading write count information from file "/tmp/test_repair_verify.data"
150172024-02-08T06:45:07.989ZRun Repair workload
150182024-02-08T06:45:07.989Z01/30 Write block 97 len 36864 data: 18 16 15 16 14 12 12 13 10
150192024-02-08T06:45:07.989Z02/30 Write block 80 len 28672 data: 10 10 11 10 10 10 10
150202024-02-08T06:45:07.989Z03/30 Read block 15 len 12288
150212024-02-08T06:45:07.993Z04/30 Write block 203 len 28672 data: 11 11 11 11 12 10 11
150222024-02-08T06:45:07.993Z05/30 Flush
150232024-02-08T06:45:07.993Z06/30 Write block 35 len 32768 data: 11 10 9 11 8 8 8 7
150242024-02-08T06:45:07.993Z07/30 Write block 369 len 32768 data: 11 11 11 15 14 6 9 11
150252024-02-08T06:45:07.993Z08/30 Write block 240 len 28672 data: 8 11 9 13 10 10 10
150262024-02-08T06:45:07.993Z09/30 Write block 509 len 4096 data: 9
150272024-02-08T06:45:07.993Z10/30 Read block 247 len 36864
150282024-02-08T06:45:07.997Z11/30 Write block 234 len 32768 data: 8 10 7 6 8 9 9 12
150292024-02-08T06:45:07.997Z12/30 Read block 541 len 24576
150302024-02-08T06:45:07.997Z13/30 Flush
150312024-02-08T06:45:08.000Z14/30 Write block 546 len 12288 data: 11 16 17
150322024-02-08T06:45:08.000Z15/30 Write block 546 len 32768 data: 12 17 18 19 17 16 14 13
150332024-02-08T06:45:08.000Z16/30 Read block 300 len 8192
150342024-02-08T06:45:08.004Z17/30 Write block 236 len 16384 data: 8 7 9 10
150352024-02-08T06:45:08.004Z18/30 Flush
150362024-02-08T06:45:08.004Z19/30 Flush
150372024-02-08T06:45:08.004Z20/30 Flush
150382024-02-08T06:45:08.007Z21/30 Write block 398 len 32768 data: 9 10 10 12 12 11 12 12
150392024-02-08T06:45:08.007Z22/30 Read block 352 len 4096
150402024-02-08T06:45:08.007Z23/30 Read block 480 len 16384
150412024-02-08T06:45:08.007Z24/30 Write block 382 len 32768 data: 8 8 12 10 10 9 10 13
150422024-02-08T06:45:08.011Z25/30 Read block 419 len 4096
150432024-02-08T06:45:08.011Z26/30 Read block 23 len 4096
150442024-02-08T06:45:08.011Z27/30 Write block 119 len 20480 data: 6 7 6 5 5
150452024-02-08T06:45:08.011Z28/30 Write block 437 len 28672 data: 16 14 10 11 11 10 11
150462024-02-08T06:45:08.011Z29/30 Write block 138 len 12288 data: 11 10 8
150472024-02-08T06:45:08.015Z30/30 Read block 566 len 16384
150482024-02-08T06:45:08.015Z----------------------------------------------------------------
150492024-02-08T06:45:08.015Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:25
150502024-02-08T06:45:08.015ZStates: Active Active Active
150512024-02-08T06:45:08.015ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
150522024-02-08T06:45:08.015Z 6 Acked 1005 Write 8 Done Done Done false
150532024-02-08T06:45:08.016Z 7 Acked 1006 Write 8 Done Done Done false
150542024-02-08T06:45:08.016Z 8 Acked 1007 Write 7 Done Done Done false
150552024-02-08T06:45:08.016Z 9 Acked 1008 Write 1 Done Done Done false
150562024-02-08T06:45:08.016Z 10 Acked 1009 Read 9 Done Done Done false
150572024-02-08T06:45:08.016Z 11 Acked 1010 Write 8 Sent Done Done false
150582024-02-08T06:45:08.016Z 12 Acked 1011 Read 6 Sent Done Done false
150592024-02-08T06:45:08.016Z 13 Acked 1012 Flush 0 Sent Done Done false
150602024-02-08T06:45:08.016Z 14 Acked 1013 Write 3 Sent Done Done false
150612024-02-08T06:45:08.016Z 15 Acked 1014 Write 8 Sent Done Done false
150622024-02-08T06:45:08.016Z 16 Acked 1015 Read 2 Sent Done Done false
150632024-02-08T06:45:08.016Z 17 Acked 1016 Write 4 Sent Done Done false
150642024-02-08T06:45:08.016Z 18 Acked 1017 Flush 0 Sent Done Done false
150652024-02-08T06:45:08.016Z 19 Acked 1018 Flush 0 Sent Done Done false
150662024-02-08T06:45:08.016Z 20 Acked 1019 Flush 0 Sent Done Done false
150672024-02-08T06:45:08.016Z 21 Acked 1020 Write 8 Sent Done Done false
150682024-02-08T06:45:08.016Z 22 Acked 1021 Read 1 Sent Done Done false
150692024-02-08T06:45:08.017Z 23 Acked 1022 Read 4 Sent Done Done false
150702024-02-08T06:45:08.017Z 24 Acked 1023 Write 8 Sent Done Done false
150712024-02-08T06:45:08.017Z 25 Acked 1024 Read 1 Sent Done Done false
150722024-02-08T06:45:08.017Z 26 Acked 1025 Read 1 Sent Done Done false
150732024-02-08T06:45:08.017Z 27 Acked 1026 Write 5 Sent Done Done false
150742024-02-08T06:45:08.017Z 28 Acked 1027 Write 7 Sent Done Done false
150752024-02-08T06:45:08.017Z 29 Acked 1028 Write 3 Sent Done Done false
150762024-02-08T06:45:08.017Z 30 Acked 1029 Read 4 Sent Done Done false
150772024-02-08T06:45:08.017Z STATES DS:0 DS:1 DS:2 TOTAL
150782024-02-08T06:45:08.017Z New 0 0 0 0
150792024-02-08T06:45:08.017Z Sent 20 0 0 20
150802024-02-08T06:45:08.017Z Done 5 25 25 55
150812024-02-08T06:45:08.017Z Skipped 0 0 0 0
150822024-02-08T06:45:08.017Z Error 0 0 0 0
150832024-02-08T06:45:08.017ZLast Flush: 1004 1019 1019
150842024-02-08T06:45:08.018ZDownstairs last five completed: 1004 1003 1002 1001 1000
150852024-02-08T06:45:08.018ZUpstairs last five completed: 30 29 28 27 26
150862024-02-08T06:45:08.018Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:07.235349078Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1690}
150872024-02-08T06:45:08.018ZWrote out file "/tmp/test_repair_verify.data"
150882024-02-08T06:45:08.021Z
150892024-02-08T06:45:08.021ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
150902024-02-08T06:45:08.039ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150912024-02-08T06:45:08.040Z 0 000-019 135 135 135 227 227 227 F F F
150922024-02-08T06:45:08.040Z 1 020-039 135 138 138 227 233 233 T F F <---
150932024-02-08T06:45:08.040Z 2 040-059 133 138 138 224 233 233 T F F <---
150942024-02-08T06:45:08.040Z 3 060-079 132 132 132 223 223 223 F F F
150952024-02-08T06:45:08.040Z 4 080-099 138 138 138 232 232 232 F F F
150962024-02-08T06:45:08.040Z 5 100-119 138 138 138 232 232 232 F T T <---
150972024-02-08T06:45:08.040Z 6 120-139 133 133 133 224 224 224 F T T <---
150982024-02-08T06:45:08.040Z 7 140-159 130 130 130 219 219 219 F T T <---
150992024-02-08T06:45:08.044Z 8 160-179 130 130 130 219 219 219 F F F
151002024-02-08T06:45:08.044Z 9 180-199 133 133 133 224 224 224 F F F
151012024-02-08T06:45:08.044Z 10 200-219 138 138 138 232 232 232 F F F
151022024-02-08T06:45:08.044Z 11 220-239 134 138 138 225 234 234 F F F <---
151032024-02-08T06:45:08.044Z 12 240-259 136 138 138 229 233 233 T F F <---
151042024-02-08T06:45:08.044Z 13 260-279 136 136 136 228 228 228 F F F
151052024-02-08T06:45:08.044Z 14 280-299 134 134 134 225 225 225 F F F
151062024-02-08T06:45:08.044Z 15 300-319 133 133 133 224 224 224 F F F
151072024-02-08T06:45:08.044Z 16 320-339 135 135 135 227 227 227 F F F
151082024-02-08T06:45:08.044Z 17 340-359 128 128 128 216 216 216 F F F
151092024-02-08T06:45:08.044Z 18 360-379 137 138 138 231 233 233 T F F <---
151102024-02-08T06:45:08.045Z 19 380-399 136 136 136 229 229 229 F T T <---
151112024-02-08T06:45:08.045Z 20 400-419 136 136 136 229 229 229 F T T <---
151122024-02-08T06:45:08.045Z 21 420-439 133 133 133 224 224 224 F T T <---
151132024-02-08T06:45:08.045Z 22 440-459 136 136 136 230 230 230 F T T <---
151142024-02-08T06:45:08.045Z 23 460-479 135 135 135 227 227 227 F F F
151152024-02-08T06:45:08.045Z 24 480-499 133 133 133 224 224 224 F F F
151162024-02-08T06:45:08.045Z 25 500-519 135 138 138 227 233 233 T F F <---
151172024-02-08T06:45:08.045Z 26 520-539 136 136 136 228 228 228 F F F
151182024-02-08T06:45:08.045Z 27 540-559 130 138 138 219 234 234 F F F <---
151192024-02-08T06:45:08.045Z 28 560-579 136 136 136 228 228 228 F F F
151202024-02-08T06:45:08.045Z 29 580-599 133 133 133 224 224 224 F F F
151212024-02-08T06:45:08.045ZMax gen: 138, Max flush: 234
151222024-02-08T06:45:08.045ZError: Difference in extent metadata found!
151232024-02-08T06:45:08.046ZA Difference in extent metadata is expected here
151242024-02-08T06:45:08.046ZOn loop 69
151252024-02-08T06:45:08.046Z
151262024-02-08T06:45:08.049ZVerifying data now
151272024-02-08T06:45:13.734ZNo change after verify
151282024-02-08T06:45:13.734ZLoop: 69 Downstairs dump after verify (and repair):
151292024-02-08T06:45:13.742ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151302024-02-08T06:45:13.742Z 0 000-019 135 135 135 227 227 227 F F F
151312024-02-08T06:45:13.743Z 1 020-039 138 138 138 233 233 233 F F F
151322024-02-08T06:45:13.743Z 2 040-059 138 138 138 233 233 233 F F F
151332024-02-08T06:45:13.743Z 3 060-079 132 132 132 223 223 223 F F F
151342024-02-08T06:45:13.743Z 4 080-099 138 138 138 232 232 232 F F F
151352024-02-08T06:45:13.743Z 5 100-119 139 139 139 235 235 235 F F F
151362024-02-08T06:45:13.743Z 6 120-139 139 139 139 235 235 235 F F F
151372024-02-08T06:45:13.743Z 7 140-159 139 139 139 235 235 235 F F F
151382024-02-08T06:45:13.743Z 8 160-179 130 130 130 219 219 219 F F F
151392024-02-08T06:45:13.743Z 9 180-199 133 133 133 224 224 224 F F F
151402024-02-08T06:45:13.743Z 10 200-219 138 138 138 232 232 232 F F F
151412024-02-08T06:45:13.743Z 11 220-239 138 138 138 234 234 234 F F F
151422024-02-08T06:45:13.743Z 12 240-259 138 138 138 233 233 233 F F F
151432024-02-08T06:45:13.743Z 13 260-279 136 136 136 228 228 228 F F F
151442024-02-08T06:45:13.747Z 14 280-299 134 134 134 225 225 225 F F F
151452024-02-08T06:45:13.747Z 15 300-319 133 133 133 224 224 224 F F F
151462024-02-08T06:45:13.747Z 16 320-339 135 135 135 227 227 227 F F F
151472024-02-08T06:45:13.747Z 17 340-359 128 128 128 216 216 216 F F F
151482024-02-08T06:45:13.747Z 18 360-379 138 138 138 233 233 233 F F F
151492024-02-08T06:45:13.748Z 19 380-399 139 139 139 235 235 235 F F F
151502024-02-08T06:45:13.748Z 20 400-419 139 139 139 235 235 235 F F F
151512024-02-08T06:45:13.748Z 21 420-439 139 139 139 235 235 235 F F F
151522024-02-08T06:45:13.748Z 22 440-459 139 139 139 235 235 235 F F F
151532024-02-08T06:45:13.748Z 23 460-479 135 135 135 227 227 227 F F F
151542024-02-08T06:45:13.748Z 24 480-499 133 133 133 224 224 224 F F F
151552024-02-08T06:45:13.748Z 25 500-519 138 138 138 233 233 233 F F F
151562024-02-08T06:45:13.748Z 26 520-539 136 136 136 228 228 228 F F F
151572024-02-08T06:45:13.748Z 27 540-559 138 138 138 234 234 234 F F F
151582024-02-08T06:45:13.748Z 28 560-579 136 136 136 228 228 228 F F F
151592024-02-08T06:45:13.748Z 29 580-599 133 133 133 224 224 224 F F F
151602024-02-08T06:45:13.748ZMax gen: 139, Max flush: 235
151612024-02-08T06:45:13.748Z
151622024-02-08T06:45:13.748ZBegin loop 70
151632024-02-08T06:45:13.749ZDownstairs to restart: 2
151642024-02-08T06:45:13.763Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.98535282Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698}
151652024-02-08T06:45:13.770Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.985629159Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698}
151662024-02-08T06:45:13.770Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
151672024-02-08T06:45:13.770Z,"time":"2024-02-08T06:45:12.985670816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698}
151682024-02-08T06:45:13.770Z{"msg":"Crucible bafe642b-76cc-4020-a450-6e5c85af1047 has session id: f2e817e8-2c51-4f0f-ac21-ff39e3bb474e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.985702174Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151692024-02-08T06:45:13.770Z{"msg":"Upstairs opts: Upstairs UUID: bafe642b-76cc-4020-a450-6e5c85af1047, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.985747071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151702024-02-08T06:45:13.770Z{"msg":"Crucible stats registered with UUID: bafe642b-76cc-4020-a450-6e5c85af1047","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.985774689Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151712024-02-08T06:45:13.770Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.985806416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698}
151722024-02-08T06:45:13.770Z{"msg":"bafe642b-76cc-4020-a450-6e5c85af1047 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.98589527Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151732024-02-08T06:45:13.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.985926987Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151742024-02-08T06:45:13.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.985958625Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151752024-02-08T06:45:13.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.98601573Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151762024-02-08T06:45:13.771Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.986039609Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151772024-02-08T06:45:13.771Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.986071566Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151782024-02-08T06:45:13.771Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.986087545Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151792024-02-08T06:45:13.771Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.986117603Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"io task","client":"2","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151802024-02-08T06:45:13.771Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.98615066Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"io task","client":"0","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151812024-02-08T06:45:13.771Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.986176378Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151822024-02-08T06:45:13.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.986212416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"io task","client":"2","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151832024-02-08T06:45:13.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.986318418Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151842024-02-08T06:45:13.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.986400742Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"io task","client":"0","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151852024-02-08T06:45:13.771Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.986861437Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151862024-02-08T06:45:13.771Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.986913043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151872024-02-08T06:45:13.771Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.987001387Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151882024-02-08T06:45:13.771Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.987066292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151892024-02-08T06:45:13.771Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.987448163Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151902024-02-08T06:45:13.772Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.987605101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151912024-02-08T06:45:13.772Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.988028679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151922024-02-08T06:45:13.772Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.988081675Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151932024-02-08T06:45:13.772Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.988126792Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151942024-02-08T06:45:13.772Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.988169709Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151952024-02-08T06:45:13.772Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.988214425Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151962024-02-08T06:45:13.772Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.988255073Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151972024-02-08T06:45:13.772Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.98828327Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151982024-02-08T06:45:13.772Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.988298679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
151992024-02-08T06:45:13.772Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.988736766Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152002024-02-08T06:45:13.772Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989178373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152012024-02-08T06:45:13.772Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.98921547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152022024-02-08T06:45:13.772Z{"msg":"[0]R flush_numbers[0..12]: [227, 233, 233, 223, 232, 235, 235, 235, 219, 224, 232, 234]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989232729Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152032024-02-08T06:45:13.772Z{"msg":"[0]R generation[0..12]: [135, 138, 138, 132, 138, 139, 139, 139, 130, 133, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989254777Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152042024-02-08T06:45:13.772Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989269786Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152052024-02-08T06:45:13.772Z{"msg":"[1]R flush_numbers[0..12]: [227, 233, 233, 223, 232, 235, 235, 235, 219, 224, 232, 234]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989291015Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152062024-02-08T06:45:13.773Z{"msg":"[1]R generation[0..12]: [135, 138, 138, 132, 138, 139, 139, 139, 130, 133, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989305303Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152072024-02-08T06:45:13.773Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989325962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152082024-02-08T06:45:13.773Z{"msg":"[2]R flush_numbers[0..12]: [227, 233, 233, 223, 232, 235, 235, 235, 219, 224, 232, 234]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989340591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152092024-02-08T06:45:13.773ZWait for a query_work_queue command to finish before sending IO
152102024-02-08T06:45:13.773Z{"msg":"[2]R generation[0..12]: [135, 138, 138, 132, 138, 139, 139, 139, 130, 133, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989373029Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152112024-02-08T06:45:13.773Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989410996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152122024-02-08T06:45:13.773Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989434004Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152132024-02-08T06:45:13.773Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989448653Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152142024-02-08T06:45:13.777Z{"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989461882Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152152024-02-08T06:45:13.777Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.98949184Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152162024-02-08T06:45:13.777Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989519758Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152172024-02-08T06:45:13.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989547065Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152182024-02-08T06:45:13.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989564764Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152192024-02-08T06:45:13.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989598581Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152202024-02-08T06:45:13.777Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.98962663Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152212024-02-08T06:45:13.777Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989651188Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152222024-02-08T06:45:13.777Z{"msg":"bafe642b-76cc-4020-a450-6e5c85af1047 is now active with session: f2e817e8-2c51-4f0f-ac21-ff39e3bb474e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989664876Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152232024-02-08T06:45:13.778Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989694584Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698,"session_id":"f2e817e8-2c51-4f0f-ac21-ff39e3bb474e"}
152242024-02-08T06:45:13.778Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:12.989727552Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698}
152252024-02-08T06:45:13.778ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
152262024-02-08T06:45:13.778ZLoading write count information from file "/tmp/test_repair_verify.data"
152272024-02-08T06:45:13.778ZRun Repair workload
152282024-02-08T06:45:13.778Z01/30 Write block 119 len 20480 data: 7 8 7 6 6
152292024-02-08T06:45:13.778Z02/30 Write block 563 len 8192 data: 14 12
152302024-02-08T06:45:13.778Z03/30 Flush
152312024-02-08T06:45:13.778Z04/30 Write block 108 len 36864 data: 11 9 5 5 6 9 9 10 10
152322024-02-08T06:45:13.778Z05/30 Write block 544 len 12288 data: 9 9 13
152332024-02-08T06:45:13.778Z06/30 Write block 15 len 8192 data: 7 6
152342024-02-08T06:45:13.778Z07/30 Flush
152352024-02-08T06:45:13.781Z08/30 Write block 346 len 20480 data: 12 10 8 8 9
152362024-02-08T06:45:13.781Z09/30 Write block 325 len 16384 data: 15 13 10 8
152372024-02-08T06:45:13.782Z10/30 Write block 249 len 4096 data: 13
152382024-02-08T06:45:13.782Z11/30 Read block 583 len 4096
152392024-02-08T06:45:13.782Z12/30 Read block 284 len 28672
152402024-02-08T06:45:13.786Z13/30 Write block 205 len 40960 data: 12 12 13 11 12 10 15 17 15 13
152412024-02-08T06:45:13.786Z14/30 Write block 111 len 12288 data: 6 7 10
152422024-02-08T06:45:13.786Z15/30 Read block 229 len 36864
152432024-02-08T06:45:13.786Z16/30 Write block 479 len 4096 data: 6
152442024-02-08T06:45:13.786Z17/30 Read block 316 len 4096
152452024-02-08T06:45:13.786Z18/30 Write block 284 len 32768 data: 9 8 7 8 10 9 13 15
152462024-02-08T06:45:13.789Z19/30 Write block 219 len 40960 data: 15 15 16 17 15 16 15 15 15 16
152472024-02-08T06:45:13.789Z20/30 Write block 476 len 24576 data: 7 6 6 7 6 7
152482024-02-08T06:45:13.789Z21/30 Read block 593 len 16384
152492024-02-08T06:45:13.789Z22/30 Read block 156 len 16384
152502024-02-08T06:45:13.792Z23/30 Write block 71 len 20480 data: 7 8 8 11 10
152512024-02-08T06:45:13.792Z24/30 Write block 167 len 40960 data: 16 13 14 12 10 12 13 9 10 9
152522024-02-08T06:45:13.792Z25/30 Read block 145 len 32768
152532024-02-08T06:45:13.793Z26/30 Read block 535 len 40960
152542024-02-08T06:45:13.796Z27/30 Write block 187 len 36864 data: 11 11 10 8 9 8 10 12 10
152552024-02-08T06:45:13.796Z28/30 Write block 116 len 16384 data: 11 8 7 8
152562024-02-08T06:45:13.796Z29/30 Read block 106 len 4096
152572024-02-08T06:45:13.796Z30/30 Write block 418 len 36864 data: 13 13 12 11 11 10 9 8 8
152582024-02-08T06:45:13.800Z----------------------------------------------------------------
152592024-02-08T06:45:13.800Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
152602024-02-08T06:45:13.800ZStates: Active Active Active
152612024-02-08T06:45:13.800ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
152622024-02-08T06:45:13.800Z 1 Acked 1000 Write 5 Done Done Done false
152632024-02-08T06:45:13.800Z 2 Acked 1001 Write 2 Done Done Sent false
152642024-02-08T06:45:13.801Z 3 Acked 1002 Flush 0 Done Done Sent false
152652024-02-08T06:45:13.801Z 4 Acked 1003 Write 9 Done Done Sent false
152662024-02-08T06:45:13.801Z 5 Acked 1004 Write 3 Done Done Sent false
152672024-02-08T06:45:13.801Z 6 Acked 1005 Write 2 Done Done Sent false
152682024-02-08T06:45:13.801Z 7 Acked 1006 Flush 0 Done Done Sent false
152692024-02-08T06:45:13.801Z 8 Acked 1007 Write 5 Done Done Sent false
152702024-02-08T06:45:13.801Z 9 Acked 1008 Write 4 Done Done Sent false
152712024-02-08T06:45:13.801Z 10 Acked 1009 Write 1 Done Done Sent false
152722024-02-08T06:45:13.801Z 11 Acked 1010 Read 1 Done Done Sent false
152732024-02-08T06:45:13.801Z 12 Acked 1011 Read 7 Done Done Sent false
152742024-02-08T06:45:13.801Z 13 Acked 1012 Write 10 Done Done Sent false
152752024-02-08T06:45:13.801Z 14 Acked 1013 Write 3 Done Done Sent false
152762024-02-08T06:45:13.801Z 15 Acked 1014 Read 9 Done Done Sent false
152772024-02-08T06:45:13.801Z 16 Acked 1015 Write 1 Done Done Sent false
152782024-02-08T06:45:13.801Z 17 Acked 1016 Read 1 Done Done Sent false
152792024-02-08T06:45:13.801Z 18 Acked 1017 Write 8 Done Done Sent false
152802024-02-08T06:45:13.802Z 19 Acked 1018 Write 10 Done Done Sent false
152812024-02-08T06:45:13.802Z 20 Acked 1019 Write 6 Done Done Sent false
152822024-02-08T06:45:13.802Z 21 Acked 1020 Read 4 Done Done Sent false
152832024-02-08T06:45:13.802Z 22 Acked 1021 Read 4 Done Done Sent false
152842024-02-08T06:45:13.802Z 23 Acked 1022 Write 5 Done Done Sent false
152852024-02-08T06:45:13.802Z 24 Acked 1023 Write 10 Done Done Sent false
152862024-02-08T06:45:13.802Z 25 Acked 1024 Read 8 Done Done Sent false
152872024-02-08T06:45:13.802Z 26 Acked 1025 Read 10 Done Done Sent false
152882024-02-08T06:45:13.802Z 27 Acked 1026 Write 9 Done Done Sent false
152892024-02-08T06:45:13.802Z 28 Acked 1027 Write 4 Done Done Sent false
152902024-02-08T06:45:13.802Z 29 Acked 1028 Read 1 Done Done Sent false
152912024-02-08T06:45:13.802Z 30 Acked 1029 Write 9 Sent Sent Sent false
152922024-02-08T06:45:13.802Z STATES DS:0 DS:1 DS:2 TOTAL
152932024-02-08T06:45:13.802Z New 0 0 0 0
152942024-02-08T06:45:13.802Z Sent 1 1 29 31
152952024-02-08T06:45:13.803Z Done 29 29 1 59
152962024-02-08T06:45:13.803Z Skipped 0 0 0 0
152972024-02-08T06:45:13.803Z Error 0 0 0 0
152982024-02-08T06:45:13.803ZLast Flush: 1006 1006 0
152992024-02-08T06:45:13.803ZDownstairs last five completed:
153002024-02-08T06:45:13.803ZUpstairs last five completed: 30 29 28 27 26
153012024-02-08T06:45:13.803Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:13.018998714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1698}
153022024-02-08T06:45:13.803ZWrote out file "/tmp/test_repair_verify.data"
153032024-02-08T06:45:13.806Z
153042024-02-08T06:45:13.806ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153052024-02-08T06:45:13.824ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153062024-02-08T06:45:13.824Z 0 000-019 140 140 135 237 237 227 F F F <---
153072024-02-08T06:45:13.824Z 1 020-039 138 138 138 233 233 233 F F F
153082024-02-08T06:45:13.824Z 2 040-059 138 138 138 233 233 233 F F F
153092024-02-08T06:45:13.824Z 3 060-079 132 132 132 223 223 223 T T F <---
153102024-02-08T06:45:13.824Z 4 080-099 138 138 138 232 232 232 F F F
153112024-02-08T06:45:13.824Z 5 100-119 140 140 139 237 237 235 T T T <---
153122024-02-08T06:45:13.824Z 6 120-139 140 140 139 236 236 235 F F T <---
153132024-02-08T06:45:13.824Z 7 140-159 139 139 139 235 235 235 F F F
153142024-02-08T06:45:13.824Z 8 160-179 130 130 130 219 219 219 T T F <---
153152024-02-08T06:45:13.827Z 9 180-199 133 133 133 224 224 224 T T F <---
153162024-02-08T06:45:13.827Z 10 200-219 138 138 138 232 232 232 T T F <---
153172024-02-08T06:45:13.828Z 11 220-239 138 138 138 234 234 234 T T F <---
153182024-02-08T06:45:13.828Z 12 240-259 138 138 138 233 233 233 T T F <---
153192024-02-08T06:45:13.828Z 13 260-279 136 136 136 228 228 228 F F F
153202024-02-08T06:45:13.828Z 14 280-299 134 134 134 225 225 225 T T F <---
153212024-02-08T06:45:13.828Z 15 300-319 133 133 133 224 224 224 F F F
153222024-02-08T06:45:13.828Z 16 320-339 135 135 135 227 227 227 T T F <---
153232024-02-08T06:45:13.828Z 17 340-359 128 128 128 216 216 216 T T F <---
153242024-02-08T06:45:13.828Z 18 360-379 138 138 138 233 233 233 F F F
153252024-02-08T06:45:13.828Z 19 380-399 139 139 139 235 235 235 F F F
153262024-02-08T06:45:13.828Z 20 400-419 139 139 139 235 235 235 T T F <---
153272024-02-08T06:45:13.828Z 21 420-439 139 139 139 235 235 235 T T F <---
153282024-02-08T06:45:13.828Z 22 440-459 139 139 139 235 235 235 F F F
153292024-02-08T06:45:13.828Z 23 460-479 135 135 135 227 227 227 T T F <---
153302024-02-08T06:45:13.828Z 24 480-499 133 133 133 224 224 224 T T F <---
153312024-02-08T06:45:13.828Z 25 500-519 138 138 138 233 233 233 F F F
153322024-02-08T06:45:13.829Z 26 520-539 136 136 136 228 228 228 F F F
153332024-02-08T06:45:13.829Z 27 540-559 140 140 138 237 237 234 F F F <---
153342024-02-08T06:45:13.829Z 28 560-579 140 140 136 236 236 228 F F F <---
153352024-02-08T06:45:13.829Z 29 580-599 133 133 133 224 224 224 F F F
153362024-02-08T06:45:13.829ZMax gen: 140, Max flush: 237
153372024-02-08T06:45:13.829ZError: Difference in extent metadata found!
153382024-02-08T06:45:13.829ZA Difference in extent metadata is expected here
153392024-02-08T06:45:13.829ZOn loop 70
153402024-02-08T06:45:13.829Z
153412024-02-08T06:45:13.832ZVerifying data now
153422024-02-08T06:45:19.682ZNo change after verify
153432024-02-08T06:45:19.682ZLoop: 70 Downstairs dump after verify (and repair):
153442024-02-08T06:45:19.698ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153452024-02-08T06:45:19.698Z 0 000-019 140 140 140 237 237 237 F F F
153462024-02-08T06:45:19.698Z 1 020-039 138 138 138 233 233 233 F F F
153472024-02-08T06:45:19.698Z 2 040-059 138 138 138 233 233 233 F F F
153482024-02-08T06:45:19.698Z 3 060-079 141 141 141 238 238 238 F F F
153492024-02-08T06:45:19.701Z 4 080-099 138 138 138 232 232 232 F F F
153502024-02-08T06:45:19.701Z 5 100-119 141 141 141 238 238 238 F F F
153512024-02-08T06:45:19.701Z 6 120-139 140 140 140 236 236 236 F F F
153522024-02-08T06:45:19.701Z 7 140-159 139 139 139 235 235 235 F F F
153532024-02-08T06:45:19.701Z 8 160-179 141 141 141 238 238 238 F F F
153542024-02-08T06:45:19.701Z 9 180-199 141 141 141 238 238 238 F F F
153552024-02-08T06:45:19.701Z 10 200-219 141 141 141 238 238 238 F F F
153562024-02-08T06:45:19.702Z 11 220-239 141 141 141 238 238 238 F F F
153572024-02-08T06:45:19.702Z 12 240-259 141 141 141 238 238 238 F F F
153582024-02-08T06:45:19.702Z 13 260-279 136 136 136 228 228 228 F F F
153592024-02-08T06:45:19.702Z 14 280-299 141 141 141 238 238 238 F F F
153602024-02-08T06:45:19.702Z 15 300-319 133 133 133 224 224 224 F F F
153612024-02-08T06:45:19.702Z 16 320-339 141 141 141 238 238 238 F F F
153622024-02-08T06:45:19.702Z 17 340-359 141 141 141 238 238 238 F F F
153632024-02-08T06:45:19.702Z 18 360-379 138 138 138 233 233 233 F F F
153642024-02-08T06:45:19.702Z 19 380-399 139 139 139 235 235 235 F F F
153652024-02-08T06:45:19.702Z 20 400-419 141 141 141 238 238 238 F F F
153662024-02-08T06:45:19.702Z 21 420-439 141 141 141 238 238 238 F F F
153672024-02-08T06:45:19.702Z 22 440-459 139 139 139 235 235 235 F F F
153682024-02-08T06:45:19.702Z 23 460-479 141 141 141 238 238 238 F F F
153692024-02-08T06:45:19.702Z 24 480-499 141 141 141 238 238 238 F F F
153702024-02-08T06:45:19.702Z 25 500-519 138 138 138 233 233 233 F F F
153712024-02-08T06:45:19.703Z 26 520-539 136 136 136 228 228 228 F F F
153722024-02-08T06:45:19.703Z 27 540-559 140 140 140 237 237 237 F F F
153732024-02-08T06:45:19.703Z 28 560-579 140 140 140 236 236 236 F F F
153742024-02-08T06:45:19.703Z 29 580-599 133 133 133 224 224 224 F F F
153752024-02-08T06:45:19.703ZMax gen: 141, Max flush: 238
153762024-02-08T06:45:19.703Z
153772024-02-08T06:45:19.703ZBegin loop 71
153782024-02-08T06:45:19.703ZDownstairs to restart: 0
153792024-02-08T06:45:19.723Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939019292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706}
153802024-02-08T06:45:19.723Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"Crucible runtime is spawned
153812024-02-08T06:45:19.724Zebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939331028Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706}
153822024-02-08T06:45:19.755Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939360816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706}
153832024-02-08T06:45:19.755Z{"msg":"Crucible 3e29d492-a20d-465a-bf22-941252074eb8 has session id: c6424719-624c-4b17-affd-5862122a2c79","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939382524Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153842024-02-08T06:45:19.755Z{"msg":"Upstairs opts: Upstairs UUID: 3e29d492-a20d-465a-bf22-941252074eb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939416232Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153852024-02-08T06:45:19.755Z{"msg":"Crucible stats registered with UUID: 3e29d492-a20d-465a-bf22-941252074eb8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939431511Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153862024-02-08T06:45:19.755Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939458049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706}
153872024-02-08T06:45:19.755Z{"msg":"3e29d492-a20d-465a-bf22-941252074eb8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939531013Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153882024-02-08T06:45:19.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939555942Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153892024-02-08T06:45:19.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939587359Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153902024-02-08T06:45:19.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939615167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153912024-02-08T06:45:19.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939640765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153922024-02-08T06:45:19.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939666963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153932024-02-08T06:45:19.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939702951Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153942024-02-08T06:45:19.755Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939724939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153952024-02-08T06:45:19.755Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939760486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153962024-02-08T06:45:19.756Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939798783Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153972024-02-08T06:45:19.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.939956292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153982024-02-08T06:45:19.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.940160536Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
153992024-02-08T06:45:19.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.940214992Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154002024-02-08T06:45:19.756Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.94064647Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154012024-02-08T06:45:19.756Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.940681347Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154022024-02-08T06:45:19.756Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.940696976Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154032024-02-08T06:45:19.756Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.940729903Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154042024-02-08T06:45:19.756Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.941338838Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154052024-02-08T06:45:19.756Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.941379595Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154062024-02-08T06:45:19.756Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.941818392Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154072024-02-08T06:45:19.756Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.941869658Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154082024-02-08T06:45:19.756Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.941992509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154092024-02-08T06:45:19.756Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942028986Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154102024-02-08T06:45:19.756Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942045315Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154112024-02-08T06:45:19.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942065043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154122024-02-08T06:45:19.757Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942079422Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154132024-02-08T06:45:19.757Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942099431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154142024-02-08T06:45:19.757Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942353652Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154152024-02-08T06:45:19.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942782159Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154162024-02-08T06:45:19.757Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942814197Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154172024-02-08T06:45:19.757Z{"msg":"[0]R flush_numbers[0..12]: [237, 233, 233, 238, 232, 238, 236, 235, 238, 238, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942833336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154182024-02-08T06:45:19.757Z{"msg":"[0]R generation[0..12]: [140, 138, 138, 141, 138, 141, 140, 139, 141, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942857594Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154192024-02-08T06:45:19.757Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942876672Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154202024-02-08T06:45:19.757Z{"msg":"[1]R flush_numbers[0..12]: [237, 233, 233, 238, 232, 238, 236, 235, 238, 238, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.9428995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154212024-02-08T06:45:19.757Z{"msg":"[1]R generation[0..12]: [140, 138, 138, 141, 138, 141, 140, 139, 141, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942917659Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154222024-02-08T06:45:19.757Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942941877Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154232024-02-08T06:45:19.757Z{"msg":"Wait for a query_work_queue command to finish before sending IO
154242024-02-08T06:45:19.757Z[2]R flush_numbers[0..12]: [237, 233, 233, 238, 232, 238, 236, 235, 238, 238, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942958246Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154252024-02-08T06:45:19.757Z{"msg":"[2]R generation[0..12]: [140, 138, 138, 141, 138, 141, 140, 139, 141, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.942981614Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154262024-02-08T06:45:19.758Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943001123Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154272024-02-08T06:45:19.758Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.94303257Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154282024-02-08T06:45:19.758Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943052729Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154292024-02-08T06:45:19.758Z{"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943072467Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154302024-02-08T06:45:19.758Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943096026Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154312024-02-08T06:45:19.758Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943119564Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154322024-02-08T06:45:19.758Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943140892Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154332024-02-08T06:45:19.758Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.94317326Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154342024-02-08T06:45:19.758Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943197728Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154352024-02-08T06:45:19.758Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943224486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154362024-02-08T06:45:19.758Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943246654Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154372024-02-08T06:45:19.758Z{"msg":"3e29d492-a20d-465a-bf22-941252074eb8 is now active with session: c6424719-624c-4b17-affd-5862122a2c79","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943274342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154382024-02-08T06:45:19.758Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943296171Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706,"session_id":"c6424719-624c-4b17-affd-5862122a2c79"}
154392024-02-08T06:45:19.758Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:18.943321909Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706}
154402024-02-08T06:45:19.758ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
154412024-02-08T06:45:19.759ZLoading write count information from file "/tmp/test_repair_verify.data"
154422024-02-08T06:45:19.759ZRun Repair workload
154432024-02-08T06:45:19.759Z01/30 Write block 113 len 24576 data: 11 10 11 12 9 8
154442024-02-08T06:45:19.759Z02/30 Read block 77 len 4096
154452024-02-08T06:45:19.759Z03/30 Write block 183 len 12288 data: 10 10 13
154462024-02-08T06:45:19.759Z04/30 Read block 271 len 36864
154472024-02-08T06:45:19.759Z05/30 Read block 542 len 32768
154482024-02-08T06:45:19.759Z06/30 Read block 374 len 20480
154492024-02-08T06:45:19.759Z07/30 Write block 513 len 8192 data: 8 6
154502024-02-08T06:45:19.759Z08/30 Write block 550 len 36864 data: 18 17 15 14 13 12 16 10 8
154512024-02-08T06:45:19.759Z09/30 Read block 227 len 28672
154522024-02-08T06:45:19.759Z10/30 Read block 359 len 24576
154532024-02-08T06:45:19.764Z11/30 Read block 62 len 8192
154542024-02-08T06:45:19.764Z12/30 Read block 421 len 24576
154552024-02-08T06:45:19.764Z13/30 Flush
154562024-02-08T06:45:19.764Z14/30 Write block 415 len 28672 data: 10 9 12 14 14 13 12
154572024-02-08T06:45:19.764Z15/30 Write block 567 len 16384 data: 12 13 11 13
154582024-02-08T06:45:19.765Z16/30 Read block 361 len 12288
154592024-02-08T06:45:19.765Z17/30 Write block 320 len 20480 data: 15 13 15 15 13
154602024-02-08T06:45:19.765Z18/30 Read block 190 len 24576
154612024-02-08T06:45:19.765Z19/30 Write block 153 len 16384 data: 10 12 12 13
154622024-02-08T06:45:19.765Z20/30 Write block 8 len 12288 data: 8 8 11
154632024-02-08T06:45:19.765Z21/30 Read block 302 len 8192
154642024-02-08T06:45:19.765Z22/30 Read block 474 len 32768
154652024-02-08T06:45:19.765Z23/30 Write block 349 len 4096 data: 9
154662024-02-08T06:45:19.765Z24/30 Write block 448 len 4096 data: 12
154672024-02-08T06:45:19.765Z25/30 Read block 489 len 28672
154682024-02-08T06:45:19.765Z26/30 Read block 534 len 20480
154692024-02-08T06:45:19.765Z27/30 Write block 213 len 20480 data: 16 14 12 12 13
154702024-02-08T06:45:19.765Z28/30 Write block 1 len 40960 data: 4 4 6 5 6 7 8 9 9 12
154712024-02-08T06:45:19.765Z29/30 Write block 335 len 32768 data: 15 17 16 17 17 15 15 14
154722024-02-08T06:45:19.765Z30/30 Write block 350 len 24576 data: 10 10 9 8 9 9
154732024-02-08T06:45:19.766Z----------------------------------------------------------------
154742024-02-08T06:45:19.766Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
154752024-02-08T06:45:19.766ZStates: Active Active Active
154762024-02-08T06:45:19.766ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
154772024-02-08T06:45:19.766Z 1 Acked 1000 Write 6 Done Done Done false
154782024-02-08T06:45:19.766Z 2 Acked 1001 Read 1 Done Done Done false
154792024-02-08T06:45:19.766Z 3 Acked 1002 Write 3 Done Done Done false
154802024-02-08T06:45:19.766Z 4 Acked 1003 Read 9 Sent Done Done false
154812024-02-08T06:45:19.766Z 5 Acked 1004 Read 8 Sent Done Done false
154822024-02-08T06:45:19.766Z 6 Acked 1005 Read 5 Sent Done Done false
154832024-02-08T06:45:19.766Z 7 Acked 1006 Write 2 Sent Done Done false
154842024-02-08T06:45:19.766Z 8 Acked 1007 Write 9 Sent Done Done false
154852024-02-08T06:45:19.766Z 9 Acked 1008 Read 7 Sent Done Done false
154862024-02-08T06:45:19.766Z 10 Acked 1009 Read 6 Sent Done Done false
154872024-02-08T06:45:19.766Z 11 Acked 1010 Read 2 Sent Done Done false
154882024-02-08T06:45:19.766Z 12 Acked 1011 Read 6 Sent Done Done false
154892024-02-08T06:45:19.766Z 13 Acked 1012 Flush 0 Sent Done Done false
154902024-02-08T06:45:19.767Z 14 Acked 1013 Write 7 Sent Done Done false
154912024-02-08T06:45:19.767Z 15 Acked 1014 Write 4 Sent Done Done false
154922024-02-08T06:45:19.767Z 16 Acked 1015 Read 3 Sent Done Done false
154932024-02-08T06:45:19.767Z 17 Acked 1016 Write 5 Sent Done Done false
154942024-02-08T06:45:19.767Z 18 Acked 1017 Read 6 Sent Done Done false
154952024-02-08T06:45:19.767Z 19 Acked 1018 Write 4 Sent Done Done false
154962024-02-08T06:45:19.767Z 20 Acked 1019 Write 3 Sent Done Done false
154972024-02-08T06:45:19.767Z 21 Acked 1020 Read 2 Sent Done Done false
154982024-02-08T06:45:19.767Z 22 Acked 1021 Read 8 Sent Done Done false
154992024-02-08T06:45:19.767Z 23 Acked 1022 Write 1 Sent Done Done false
155002024-02-08T06:45:19.767Z 24 Acked 1023 Write 1 Sent Done Done false
155012024-02-08T06:45:19.767Z 25 Acked 1024 Read 7 Sent Done Done false
155022024-02-08T06:45:19.767Z 26 Acked 1025 Read 5 Sent Done Done false
155032024-02-08T06:45:19.767Z 27 Acked 1026 Write 5 Sent Done Done false
155042024-02-08T06:45:19.767Z 28 Acked 1027 Write 10 Sent Done Done false
155052024-02-08T06:45:19.767Z 29 Acked 1028 Write 8 Sent Sent Done false
155062024-02-08T06:45:19.768Z 30 Acked 1029 Write 6 Sent Sent Sent false
155072024-02-08T06:45:19.768Z STATES DS:0 DS:1 DS:2 TOTAL
155082024-02-08T06:45:19.768Z New 0 0 0 0
155092024-02-08T06:45:19.768Z Sent 27 2 1 30
155102024-02-08T06:45:19.768Z Done 3 28 29 60
155112024-02-08T06:45:19.768Z Skipped 0 0 0 0
155122024-02-08T06:45:19.768Z Error 0 0 0 0
155132024-02-08T06:45:19.768ZLast Flush: 0 1012 1012
155142024-02-08T06:45:19.768ZDownstairs last five completed:
155152024-02-08T06:45:19.768ZUpstairs last five completed: 30 29 28 27 26
155162024-02-08T06:45:19.768Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:18.970666616Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1706}
155172024-02-08T06:45:19.768ZWrote out file "/tmp/test_repair_verify.data"
155182024-02-08T06:45:19.768Z
155192024-02-08T06:45:19.768ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155202024-02-08T06:45:19.774ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155212024-02-08T06:45:19.774Z 0 000-019 140 140 140 237 237 237 F T T <---
155222024-02-08T06:45:19.774Z 1 020-039 138 138 138 233 233 233 F F F
155232024-02-08T06:45:19.774Z 2 040-059 138 138 138 233 233 233 F F F
155242024-02-08T06:45:19.774Z 3 060-079 141 141 141 238 238 238 F F F
155252024-02-08T06:45:19.774Z 4 080-099 138 138 138 232 232 232 F F F
155262024-02-08T06:45:19.774Z 5 100-119 141 142 142 238 239 239 T F F <---
155272024-02-08T06:45:19.774Z 6 120-139 140 140 140 236 236 236 F F F
155282024-02-08T06:45:19.775Z 7 140-159 139 139 139 235 235 235 F T T <---
155292024-02-08T06:45:19.778Z 8 160-179 141 141 141 238 238 238 F F F
155302024-02-08T06:45:19.778Z 9 180-199 141 142 142 238 239 239 T F F <---
155312024-02-08T06:45:19.778Z 10 200-219 141 141 141 238 238 238 F T T <---
155322024-02-08T06:45:19.778Z 11 220-239 141 141 141 238 238 238 F F F
155332024-02-08T06:45:19.778Z 12 240-259 141 141 141 238 238 238 F F F
155342024-02-08T06:45:19.778Z 13 260-279 136 136 136 228 228 228 F F F
155352024-02-08T06:45:19.778Z 14 280-299 141 141 141 238 238 238 F F F
155362024-02-08T06:45:19.778Z 15 300-319 133 133 133 224 224 224 F F F
155372024-02-08T06:45:19.778Z 16 320-339 141 141 141 238 238 238 F T T <---
155382024-02-08T06:45:19.778Z 17 340-359 141 141 141 238 238 238 F T T <---
155392024-02-08T06:45:19.779Z 18 360-379 138 138 138 233 233 233 F F F
155402024-02-08T06:45:19.779Z 19 380-399 139 139 139 235 235 235 F F F
155412024-02-08T06:45:19.779Z 20 400-419 141 141 141 238 238 238 F T T <---
155422024-02-08T06:45:19.779Z 21 420-439 141 141 141 238 238 238 F T T <---
155432024-02-08T06:45:19.779Z 22 440-459 139 139 139 235 235 235 F T T <---
155442024-02-08T06:45:19.779Z 23 460-479 141 141 141 238 238 238 F F F
155452024-02-08T06:45:19.779Z 24 480-499 141 141 141 238 238 238 F F F
155462024-02-08T06:45:19.779Z 25 500-519 138 142 142 233 239 239 F F F <---
155472024-02-08T06:45:19.779Z 26 520-539 136 136 136 228 228 228 F F F
155482024-02-08T06:45:19.779Z 27 540-559 140 142 142 237 239 239 F F F <---
155492024-02-08T06:45:19.779Z 28 560-579 140 140 140 236 236 236 F T T <---
155502024-02-08T06:45:19.779Z 29 580-599 133 133 133 224 224 224 F F F
155512024-02-08T06:45:19.779ZMax gen: 142, Max flush: 239
155522024-02-08T06:45:19.779ZError: Difference in extent metadata found!
155532024-02-08T06:45:19.779ZA Difference in extent metadata is expected here
155542024-02-08T06:45:19.780ZOn loop 71
155552024-02-08T06:45:19.780Z
155562024-02-08T06:45:19.783ZVerifying data now
155572024-02-08T06:45:25.427ZNo change after verify
155582024-02-08T06:45:25.430ZLoop: 71 Downstairs dump after verify (and repair):
155592024-02-08T06:45:25.442ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155602024-02-08T06:45:25.442Z 0 000-019 143 143 143 240 240 240 F F F
155612024-02-08T06:45:25.442Z 1 020-039 138 138 138 233 233 233 F F F
155622024-02-08T06:45:25.442Z 2 040-059 138 138 138 233 233 233 F F F
155632024-02-08T06:45:25.443Z 3 060-079 141 141 141 238 238 238 F F F
155642024-02-08T06:45:25.443Z 4 080-099 138 138 138 232 232 232 F F F
155652024-02-08T06:45:25.446Z 5 100-119 142 142 142 239 239 239 F F F
155662024-02-08T06:45:25.446Z 6 120-139 140 140 140 236 236 236 F F F
155672024-02-08T06:45:25.446Z 7 140-159 143 143 143 240 240 240 F F F
155682024-02-08T06:45:25.446Z 8 160-179 141 141 141 238 238 238 F F F
155692024-02-08T06:45:25.447Z 9 180-199 142 142 142 239 239 239 F F F
155702024-02-08T06:45:25.447Z 10 200-219 143 143 143 240 240 240 F F F
155712024-02-08T06:45:25.447Z 11 220-239 141 141 141 238 238 238 F F F
155722024-02-08T06:45:25.447Z 12 240-259 141 141 141 238 238 238 F F F
155732024-02-08T06:45:25.447Z 13 260-279 136 136 136 228 228 228 F F F
155742024-02-08T06:45:25.447Z 14 280-299 141 141 141 238 238 238 F F F
155752024-02-08T06:45:25.447Z 15 300-319 133 133 133 224 224 224 F F F
155762024-02-08T06:45:25.447Z 16 320-339 143 143 143 240 240 240 F F F
155772024-02-08T06:45:25.447Z 17 340-359 143 143 143 240 240 240 F F F
155782024-02-08T06:45:25.447Z 18 360-379 138 138 138 233 233 233 F F F
155792024-02-08T06:45:25.447Z 19 380-399 139 139 139 235 235 235 F F F
155802024-02-08T06:45:25.447Z 20 400-419 143 143 143 240 240 240 F F F
155812024-02-08T06:45:25.447Z 21 420-439 143 143 143 240 240 240 F F F
155822024-02-08T06:45:25.447Z 22 440-459 143 143 143 240 240 240 F F F
155832024-02-08T06:45:25.447Z 23 460-479 141 141 141 238 238 238 F F F
155842024-02-08T06:45:25.447Z 24 480-499 141 141 141 238 238 238 F F F
155852024-02-08T06:45:25.448Z 25 500-519 142 142 142 239 239 239 F F F
155862024-02-08T06:45:25.448Z 26 520-539 136 136 136 228 228 228 F F F
155872024-02-08T06:45:25.448Z 27 540-559 142 142 142 239 239 239 F F F
155882024-02-08T06:45:25.448Z 28 560-579 143 143 143 240 240 240 F F F
155892024-02-08T06:45:25.448Z 29 580-599 133 133 133 224 224 224 F F F
155902024-02-08T06:45:25.448ZMax gen: 143, Max flush: 240
155912024-02-08T06:45:25.448Z
155922024-02-08T06:45:25.448ZBegin loop 72
155932024-02-08T06:45:25.448ZDownstairs to restart: 0
155942024-02-08T06:45:25.466Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.683735101Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714}
155952024-02-08T06:45:25.473Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.683985322Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714}
155962024-02-08T06:45:25.473Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684036738Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714}
155972024-02-08T06:45:25.473Z{"msg":"Crucible 8d5c605c-4c79-4042-99ba-08cc01a831d0 has session id: 0a4d12ba-ed70-499d-8e48-bb100d62c513","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684061117Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
155982024-02-08T06:45:25.473ZCrucible runtime is spawned
155992024-02-08T06:45:25.473Z{"msg":"Upstairs opts: Upstairs UUID: 8d5c605c-4c79-4042-99ba-08cc01a831d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684104803Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156002024-02-08T06:45:25.473Z{"msg":"Crucible stats registered with UUID: 8d5c605c-4c79-4042-99ba-08cc01a831d0","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684136411Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156012024-02-08T06:45:25.473Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.68415557Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714}
156022024-02-08T06:45:25.473Z{"msg":"8d5c605c-4c79-4042-99ba-08cc01a831d0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684320837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156032024-02-08T06:45:25.473Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684348075Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156042024-02-08T06:45:25.473Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684369194Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156052024-02-08T06:45:25.473Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684384012Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156062024-02-08T06:45:25.474Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684402131Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156072024-02-08T06:45:25.474Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684426919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156082024-02-08T06:45:25.474Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684453207Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156092024-02-08T06:45:25.474Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684488074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"io task","client":"0","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156102024-02-08T06:45:25.474Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684505093Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"io task","client":"2","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156112024-02-08T06:45:25.474Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684538881Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156122024-02-08T06:45:25.474Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684784342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156132024-02-08T06:45:25.474Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684824039Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"io task","client":"0","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156142024-02-08T06:45:25.474Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.684865316Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"io task","client":"2","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156152024-02-08T06:45:25.474Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.685577483Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156162024-02-08T06:45:25.474Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.68561392Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156172024-02-08T06:45:25.474Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.685654547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156182024-02-08T06:45:25.474Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.685697754Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156192024-02-08T06:45:25.474Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.686245202Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156202024-02-08T06:45:25.474Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.686297409Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156212024-02-08T06:45:25.475Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.68681071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156222024-02-08T06:45:25.475Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.686838568Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156232024-02-08T06:45:25.475Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.686854937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156242024-02-08T06:45:25.475Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.686886665Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156252024-02-08T06:45:25.475Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.686911283Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156262024-02-08T06:45:25.475Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.68695172Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156272024-02-08T06:45:25.475Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.686976478Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156282024-02-08T06:45:25.475Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.687019444Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156292024-02-08T06:45:25.475Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.687399906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156302024-02-08T06:45:25.475Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.687831583Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156312024-02-08T06:45:25.475Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.687894159Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156322024-02-08T06:45:25.475Z{"msg":"[0]R flush_numbers[0..12]: [240, 233, 233, 238, 232, 239, 236, 240, 238, 239, 240, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.687928426Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156332024-02-08T06:45:25.475Z{"msg":"[0]R generation[0..12]: [143, 138, 138, 141, 138, 142, 140, 143, 141, 142, 143, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.687954144Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156342024-02-08T06:45:25.475Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.687986412Z"Wait for a query_work_queue command to finish before sending IO
156352024-02-08T06:45:25.475Z,"hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156362024-02-08T06:45:25.476Z{"msg":"[1]R flush_numbers[0..12]: [240, 233, 233, 238, 232, 239, 236, 240, 238, 239, 240, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688052547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156372024-02-08T06:45:25.476Z{"msg":"[1]R generation[0..12]: [143, 138, 138, 141, 138, 142, 140, 143, 141, 142, 143, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688082905Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156382024-02-08T06:45:25.476Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688108813Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156392024-02-08T06:45:25.476Z{"msg":"[2]R flush_numbers[0..12]: [240, 233, 233, 238, 232, 239, 236, 240, 238, 239, 240, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.68814032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156402024-02-08T06:45:25.476Z{"msg":"[2]R generation[0..12]: [143, 138, 138, 141, 138, 142, 140, 143, 141, 142, 143, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688166398Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156412024-02-08T06:45:25.476Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688196106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156422024-02-08T06:45:25.476Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688220214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156432024-02-08T06:45:25.476Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688249352Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156442024-02-08T06:45:25.476Z{"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.68827275Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156452024-02-08T06:45:25.476Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688299358Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156462024-02-08T06:45:25.476Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688322837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156472024-02-08T06:45:25.476Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688348065Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156482024-02-08T06:45:25.476Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688371853Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156492024-02-08T06:45:25.476Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688398381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156502024-02-08T06:45:25.476Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688421619Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156512024-02-08T06:45:25.477Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688447407Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156522024-02-08T06:45:25.477Z{"msg":"8d5c605c-4c79-4042-99ba-08cc01a831d0 is now active with session: 0a4d12ba-ed70-499d-8e48-bb100d62c513","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688470435Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156532024-02-08T06:45:25.477Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688495514Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714,"session_id":"0a4d12ba-ed70-499d-8e48-bb100d62c513"}
156542024-02-08T06:45:25.477Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:24.688517462Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714}
156552024-02-08T06:45:25.477ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
156562024-02-08T06:45:25.477ZLoading write count information from file "/tmp/test_repair_verify.data"
156572024-02-08T06:45:25.477ZRun Repair workload
156582024-02-08T06:45:25.477Z01/30 Write block 155 len 36864 data: 13 14 11 12 11 9 10 12 12
156592024-02-08T06:45:25.481Z02/30 Read block 377 len 12288
156602024-02-08T06:45:25.481Z03/30 Read block 279 len 32768
156612024-02-08T06:45:25.481Z04/30 Read block 182 len 4096
156622024-02-08T06:45:25.481Z05/30 Read block 389 len 36864
156632024-02-08T06:45:25.481Z06/30 Write block 74 len 28672 data: 12 11 11 8 9 10 11
156642024-02-08T06:45:25.481Z07/30 Flush
156652024-02-08T06:45:25.482Z08/30 Write block 123 len 40960 data: 7 7 7 5 4 6 7 8 10 10
156662024-02-08T06:45:25.482Z09/30 Read block 449 len 12288
156672024-02-08T06:45:25.482Z10/30 Read block 490 len 12288
156682024-02-08T06:45:25.482Z11/30 Read block 72 len 36864
156692024-02-08T06:45:25.482Z12/30 Write block 284 len 20480 data: 10 9 8 9 11
156702024-02-08T06:45:25.485Z13/30 Read block 435 len 24576
156712024-02-08T06:45:25.485Z14/30 Flush
156722024-02-08T06:45:25.485Z15/30 Write block 109 len 12288 data: 10 6 7
156732024-02-08T06:45:25.485Z16/30 Flush
156742024-02-08T06:45:25.488Z17/30 Write block 128 len 28672 data: 7 8 9 11 11 10 9
156752024-02-08T06:45:25.489Z18/30 Read block 161 len 12288
156762024-02-08T06:45:25.491Z19/30 Write block 550 len 24576 data: 19 18 16 15 14 13
156772024-02-08T06:45:25.492Z20/30 Read block 61 len 16384
156782024-02-08T06:45:25.492Z21/30 Flush
156792024-02-08T06:45:25.495Z22/30 Write block 63 len 36864 data: 9 7 7 6 7 12 13 8 8
156802024-02-08T06:45:25.495Z23/30 Read block 550 len 40960
156812024-02-08T06:45:25.498Z24/30 Flush
156822024-02-08T06:45:25.501Z25/30 Write block 555 len 32768 data: 14 17 11 9 11 13 13 14
156832024-02-08T06:45:25.501Z26/30 Write block 149 len 28672 data: 10 8 10 9 11 13 14
156842024-02-08T06:45:25.501Z27/30 Read block 252 len 16384
156852024-02-08T06:45:25.504Z28/30 Read block 527 len 20480
156862024-02-08T06:45:25.504Z29/30 Write block 484 len 32768 data: 10 11 13 12 12 10 10 11
156872024-02-08T06:45:25.504Z30/30 Write block 160 len 36864 data: 10 11 13 13 12 13 14 17 14
156882024-02-08T06:45:25.507Z----------------------------------------------------------------
156892024-02-08T06:45:25.507Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
156902024-02-08T06:45:25.507ZStates: Active Active Active
156912024-02-08T06:45:25.508ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
156922024-02-08T06:45:25.508Z 1 Acked 1000 Write 9 Done Done Done false
156932024-02-08T06:45:25.508Z 2 Acked 1001 Read 3 Done Done Done false
156942024-02-08T06:45:25.508Z 3 Acked 1002 Read 8 Sent Done Done false
156952024-02-08T06:45:25.508Z 4 Acked 1003 Read 1 Sent Done Done false
156962024-02-08T06:45:25.508Z 5 Acked 1004 Read 9 Sent Done Done false
156972024-02-08T06:45:25.508Z 6 Acked 1005 Write 7 Sent Done Done false
156982024-02-08T06:45:25.508Z 7 Acked 1006 Flush 0 Sent Done Done false
156992024-02-08T06:45:25.508Z 8 Acked 1007 Write 10 Sent Done Done false
157002024-02-08T06:45:25.508Z 9 Acked 1008 Read 3 Sent Done Done false
157012024-02-08T06:45:25.508Z 10 Acked 1009 Read 3 Sent Done Done false
157022024-02-08T06:45:25.508Z 11 Acked 1010 Read 9 Sent Done Done false
157032024-02-08T06:45:25.508Z 12 Acked 1011 Write 5 Sent Done Done false
157042024-02-08T06:45:25.508Z 13 Acked 1012 Read 6 Sent Done Done false
157052024-02-08T06:45:25.508Z 14 Acked 1013 Flush 0 Sent Done Done false
157062024-02-08T06:45:25.508Z 15 Acked 1014 Write 3 Sent Done Done false
157072024-02-08T06:45:25.509Z 16 Acked 1015 Flush 0 Sent Done Done false
157082024-02-08T06:45:25.509Z 17 Acked 1016 Write 7 Sent Done Done false
157092024-02-08T06:45:25.509Z 18 Acked 1017 Read 3 Sent Done Done false
157102024-02-08T06:45:25.509Z 19 Acked 1018 Write 6 Sent Done Done false
157112024-02-08T06:45:25.509Z 20 Acked 1019 Read 4 Sent Done Done false
157122024-02-08T06:45:25.509Z 21 Acked 1020 Flush 0 Sent Done Done false
157132024-02-08T06:45:25.509Z 22 Acked 1021 Write 9 Sent Done Done false
157142024-02-08T06:45:25.509Z 23 Acked 1022 Read 10 Sent Done Done false
157152024-02-08T06:45:25.509Z 24 Acked 1023 Flush 0 Sent Done Done false
157162024-02-08T06:45:25.509Z 25 Acked 1024 Write 8 Sent Done Done false
157172024-02-08T06:45:25.509Z 26 Acked 1025 Write 7 Sent Done Done false
157182024-02-08T06:45:25.509Z 27 Acked 1026 Read 4 Sent Done Done false
157192024-02-08T06:45:25.509Z 28 Acked 1027 Read 5 Sent Done Done false
157202024-02-08T06:45:25.509Z 29 Acked 1028 Write 8 Sent Done Done false
157212024-02-08T06:45:25.509Z 30 Acked 1029 Write 9 Sent Sent Sent false
157222024-02-08T06:45:25.510Z STATES DS:0 DS:1 DS:2 TOTAL
157232024-02-08T06:45:25.510Z New 0 0 0 0
157242024-02-08T06:45:25.510Z Sent 28 1 1 30
157252024-02-08T06:45:25.510Z Done 2 29 29 60
157262024-02-08T06:45:25.510Z Skipped 0 0 0 0
157272024-02-08T06:45:25.510Z Error 0 0 0 0
157282024-02-08T06:45:25.510ZLast Flush: 0 1023 1023
157292024-02-08T06:45:25.510ZDownstairs last five completed:
157302024-02-08T06:45:25.510ZUpstairs last five completed: 30 29 28 27 26
157312024-02-08T06:45:25.510Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:24.725359586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1714}
157322024-02-08T06:45:25.510ZWrote out file "/tmp/test_repair_verify.data"
157332024-02-08T06:45:25.514Z
157342024-02-08T06:45:25.514ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
157352024-02-08T06:45:25.534ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157362024-02-08T06:45:25.534Z 0 000-019 143 143 143 240 240 240 F F F
157372024-02-08T06:45:25.535Z 1 020-039 138 138 138 233 233 233 F F F
157382024-02-08T06:45:25.535Z 2 040-059 138 138 138 233 233 233 F F F
157392024-02-08T06:45:25.535Z 3 060-079 141 144 144 238 245 245 F F F <---
157402024-02-08T06:45:25.535Z 4 080-099 138 144 144 232 241 241 F F F <---
157412024-02-08T06:45:25.535Z 5 100-119 142 144 144 239 243 243 F F F <---
157422024-02-08T06:45:25.545Z 6 120-139 140 144 144 236 244 244 F F F <---
157432024-02-08T06:45:25.545Z 7 140-159 143 144 144 240 241 241 T T T <---
157442024-02-08T06:45:25.545Z 8 160-179 141 144 144 238 241 241 T T T <---
157452024-02-08T06:45:25.545Z 9 180-199 142 142 142 239 239 239 F F F
157462024-02-08T06:45:25.545Z 10 200-219 143 143 143 240 240 240 F F F
157472024-02-08T06:45:25.546Z 11 220-239 141 141 141 238 238 238 F F F
157482024-02-08T06:45:25.546Z 12 240-259 141 141 141 238 238 238 F F F
157492024-02-08T06:45:25.546Z 13 260-279 136 136 136 228 228 228 F F F
157502024-02-08T06:45:25.546Z 14 280-299 141 144 144 238 242 242 F F F <---
157512024-02-08T06:45:25.546Z 15 300-319 133 133 133 224 224 224 F F F
157522024-02-08T06:45:25.546Z 16 320-339 143 143 143 240 240 240 F F F
157532024-02-08T06:45:25.546Z 17 340-359 143 143 143 240 240 240 F F F
157542024-02-08T06:45:25.546Z 18 360-379 138 138 138 233 233 233 F F F
157552024-02-08T06:45:25.546Z 19 380-399 139 139 139 235 235 235 F F F
157562024-02-08T06:45:25.546Z 20 400-419 143 143 143 240 240 240 F F F
157572024-02-08T06:45:25.546Z 21 420-439 143 143 143 240 240 240 F F F
157582024-02-08T06:45:25.546Z 22 440-459 143 143 143 240 240 240 F F F
157592024-02-08T06:45:25.546Z 23 460-479 141 141 141 238 238 238 F F F
157602024-02-08T06:45:25.546Z 24 480-499 141 141 141 238 238 238 F T T <---
157612024-02-08T06:45:25.546Z 25 500-519 142 142 142 239 239 239 F F F
157622024-02-08T06:45:25.546Z 26 520-539 136 136 136 228 228 228 F F F
157632024-02-08T06:45:25.547Z 27 540-559 142 144 144 239 244 244 F T T <---
157642024-02-08T06:45:25.547Z 28 560-579 143 143 143 240 240 240 F T T <---
157652024-02-08T06:45:25.547Z 29 580-599 133 133 133 224 224 224 F F F
157662024-02-08T06:45:25.547ZMax gen: 144, Max flush: 245
157672024-02-08T06:45:25.547ZError: Difference in extent metadata found!
157682024-02-08T06:45:25.547ZA Difference in extent metadata is expected here
157692024-02-08T06:45:25.547ZOn loop 72
157702024-02-08T06:45:25.547Z
157712024-02-08T06:45:25.547ZVerifying data now
157722024-02-08T06:45:31.042ZNo change after verify
157732024-02-08T06:45:31.043ZLoop: 72 Downstairs dump after verify (and repair):
157742024-02-08T06:45:31.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157752024-02-08T06:45:31.057Z 0 000-019 143 143 143 240 240 240 F F F
157762024-02-08T06:45:31.057Z 1 020-039 138 138 138 233 233 233 F F F
157772024-02-08T06:45:31.057Z 2 040-059 138 138 138 233 233 233 F F F
157782024-02-08T06:45:31.057Z 3 060-079 144 144 144 245 245 245 F F F
157792024-02-08T06:45:31.057Z 4 080-099 144 144 144 241 241 241 F F F
157802024-02-08T06:45:31.061Z 5 100-119 144 144 144 243 243 243 F F F
157812024-02-08T06:45:31.061Z 6 120-139 144 144 144 244 244 244 F F F
157822024-02-08T06:45:31.061Z 7 140-159 145 145 145 246 246 246 F F F
157832024-02-08T06:45:31.061Z 8 160-179 145 145 145 246 246 246 F F F
157842024-02-08T06:45:31.061Z 9 180-199 142 142 142 239 239 239 F F F
157852024-02-08T06:45:31.061Z 10 200-219 143 143 143 240 240 240 F F F
157862024-02-08T06:45:31.061Z 11 220-239 141 141 141 238 238 238 F F F
157872024-02-08T06:45:31.061Z 12 240-259 141 141 141 238 238 238 F F F
157882024-02-08T06:45:31.062Z 13 260-279 136 136 136 228 228 228 F F F
157892024-02-08T06:45:31.062Z 14 280-299 144 144 144 242 242 242 F F F
157902024-02-08T06:45:31.062Z 15 300-319 133 133 133 224 224 224 F F F
157912024-02-08T06:45:31.062Z 16 320-339 143 143 143 240 240 240 F F F
157922024-02-08T06:45:31.062Z 17 340-359 143 143 143 240 240 240 F F F
157932024-02-08T06:45:31.062Z 18 360-379 138 138 138 233 233 233 F F F
157942024-02-08T06:45:31.062Z 19 380-399 139 139 139 235 235 235 F F F
157952024-02-08T06:45:31.062Z 20 400-419 143 143 143 240 240 240 F F F
157962024-02-08T06:45:31.062Z 21 420-439 143 143 143 240 240 240 F F F
157972024-02-08T06:45:31.062Z 22 440-459 143 143 143 240 240 240 F F F
157982024-02-08T06:45:31.062Z 23 460-479 141 141 141 238 238 238 F F F
157992024-02-08T06:45:31.062Z 24 480-499 145 145 145 246 246 246 F F F
158002024-02-08T06:45:31.062Z 25 500-519 142 142 142 239 239 239 F F F
158012024-02-08T06:45:31.062Z 26 520-539 136 136 136 228 228 228 F F F
158022024-02-08T06:45:31.062Z 27 540-559 145 145 145 246 246 246 F F F
158032024-02-08T06:45:31.062Z 28 560-579 145 145 145 246 246 246 F F F
158042024-02-08T06:45:31.063Z 29 580-599 133 133 133 224 224 224 F F F
158052024-02-08T06:45:31.063ZMax gen: 145, Max flush: 246
158062024-02-08T06:45:31.063Z
158072024-02-08T06:45:31.063ZBegin loop 73
158082024-02-08T06:45:31.063ZDownstairs to restart: 1
158092024-02-08T06:45:31.078Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298276113Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722}
158102024-02-08T06:45:31.078Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298509265Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722}
158112024-02-08T06:45:31.078Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298530154Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722}
158122024-02-08T06:45:31.084Z{"msg":"Crucible e8a64f5a-d7d0-4baf-8c94-d4fe2cedd710 has session id: 692787f2-c710-44bf-ad51-eb7e7e776d0f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298559281Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal",Crucible runtime is spawned
158132024-02-08T06:45:31.084Z"pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158142024-02-08T06:45:31.084Z{"msg":"Upstairs opts: Upstairs UUID: e8a64f5a-d7d0-4baf-8c94-d4fe2cedd710, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298615287Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158152024-02-08T06:45:31.084Z{"msg":"Crucible stats registered with UUID: e8a64f5a-d7d0-4baf-8c94-d4fe2cedd710","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298655824Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158162024-02-08T06:45:31.084Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298681482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722}
158172024-02-08T06:45:31.084Z{"msg":"e8a64f5a-d7d0-4baf-8c94-d4fe2cedd710 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298722449Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158182024-02-08T06:45:31.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298749267Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158192024-02-08T06:45:31.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298780825Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158202024-02-08T06:45:31.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298799953Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158212024-02-08T06:45:31.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298833681Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158222024-02-08T06:45:31.085Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298860179Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158232024-02-08T06:45:31.085Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298881817Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158242024-02-08T06:45:31.085Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298913125Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"io task","client":"0","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158252024-02-08T06:45:31.085Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298929044Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158262024-02-08T06:45:31.085Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.298962191Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158272024-02-08T06:45:31.085Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:30.299000448Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158282024-02-08T06:45:31.085Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:30.299125699Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158292024-02-08T06:45:31.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.299174645Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158302024-02-08T06:45:31.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.299205473Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"io task","client":"0","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158312024-02-08T06:45:31.085Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:30.29924738Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158322024-02-08T06:45:31.085Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.299288167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158332024-02-08T06:45:31.085Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.299314845Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158342024-02-08T06:45:31.085Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.299341353Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158352024-02-08T06:45:31.085Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.299787579Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158362024-02-08T06:45:31.085Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.299826206Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158372024-02-08T06:45:31.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.29990864Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158382024-02-08T06:45:31.086Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.299935558Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158392024-02-08T06:45:31.086Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.30045105Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158402024-02-08T06:45:31.086Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.300646885Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158412024-02-08T06:45:31.086Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.300899396Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158422024-02-08T06:45:31.086Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.300942653Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158432024-02-08T06:45:31.086Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.30096992Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158442024-02-08T06:45:31.086Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.301182144Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158452024-02-08T06:45:31.086Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.301228541Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158462024-02-08T06:45:31.086Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:30.301255449Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158472024-02-08T06:45:32.080Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.299877164Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158482024-02-08T06:45:32.086Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.300090679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158492024-02-08T06:45:32.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.301032458Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158502024-02-08T06:45:32.086Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.301068425Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158512024-02-08T06:45:32.086Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.301638602Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158522024-02-08T06:45:32.086Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302214679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158532024-02-08T06:45:32.086Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302251906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158542024-02-08T06:45:32.086Z{"msg":"[0]R flush_numbers[0..12]: [240, 233, 233, 245, 241, 243, 244, 246, 246, 239, 240, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302286414Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158552024-02-08T06:45:32.087Z{"msg":"[0]R generation[0..12]: [143, 138, 138, 144, 144, 144, 144, 145, 145, 142, 143, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302310242Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158562024-02-08T06:45:32.087Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302325701Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158572024-02-08T06:45:32.087Z{"msg":"[1]R flush_numbers[0..12]: [240, 233, 233, 245, 241, 243, 244, 246, 246, 239, 240, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302347769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158582024-02-08T06:45:32.087Z{"msg":"[1]R generation[0..12]: [143, 138, 138, 144, 144, 144, 144, 145, 145, 142, 143, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302362498Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158592024-02-08T06:45:32.087Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302383186Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158602024-02-08T06:45:32.087Z{"msg":"[2]R flush_numbers[0..12]: [240, 233, 233, 245, 241, 243, 244, 246, 246, 239, 240, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302397405Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158612024-02-08T06:45:32.087Z{"msg":"[2]R generation[0..12]: [143, 138, 138, 144, 144, 144, 144, 145, 145, 142, 143, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302418654Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158622024-02-08T06:45:32.087Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
158632024-02-08T06:45:32.087Z,"time":"2024-02-08T06:45:31.302433163Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158642024-02-08T06:45:32.087Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302454221Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158652024-02-08T06:45:32.087Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.30246837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158662024-02-08T06:45:32.087Z{"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302488268Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158672024-02-08T06:45:32.087Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302501767Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158682024-02-08T06:45:32.087Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302527996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158692024-02-08T06:45:32.087Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302548054Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158702024-02-08T06:45:32.088Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302562503Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158712024-02-08T06:45:32.088Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302583241Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158722024-02-08T06:45:32.088Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.30259782Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158732024-02-08T06:45:32.088Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302617539Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158742024-02-08T06:45:32.088Z{"msg":"e8a64f5a-d7d0-4baf-8c94-d4fe2cedd710 is now active with session: 692787f2-c710-44bf-ad51-eb7e7e776d0f","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302630708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158752024-02-08T06:45:32.088Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302642847Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722,"session_id":"692787f2-c710-44bf-ad51-eb7e7e776d0f"}
158762024-02-08T06:45:32.088Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:31.302665275Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722}
158772024-02-08T06:45:32.088ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
158782024-02-08T06:45:32.088ZLoading write count information from file "/tmp/test_repair_verify.data"
158792024-02-08T06:45:32.088ZRun Repair workload
158802024-02-08T06:45:32.088Z01/30 Write block 549 len 32768 data: 20 20 19 17 16 15 15 18
158812024-02-08T06:45:32.092Z02/30 Read block 381 len 32768
158822024-02-08T06:45:32.092Z03/30 Read block 105 len 40960
158832024-02-08T06:45:32.092Z04/30 Flush
158842024-02-08T06:45:32.092Z05/30 Write block 346 len 4096 data: 13
158852024-02-08T06:45:32.092Z06/30 Read block 510 len 28672
158862024-02-08T06:45:32.092Z07/30 Read block 13 len 20480
158872024-02-08T06:45:32.095Z08/30 Read block 47 len 16384
158882024-02-08T06:45:32.095Z09/30 Read block 194 len 32768
158892024-02-08T06:45:32.095Z10/30 Flush
158902024-02-08T06:45:32.099Z11/30 Write block 442 len 24576 data: 11 12 11 12 10 11
158912024-02-08T06:45:32.099Z12/30 Flush
158922024-02-08T06:45:32.099Z13/30 Write block 205 len 32768 data: 13 13 14 12 13 11 16 18
158932024-02-08T06:45:32.101Z14/30 Read block 394 len 16384
158942024-02-08T06:45:32.102Z15/30 Write block 472 len 28672 data: 11 10 12 11 8 7 7
158952024-02-08T06:45:32.102Z16/30 Read block 87 len 32768
158962024-02-08T06:45:32.105Z17/30 Write block 532 len 32768 data: 14 13 13 13 13 11 10 8
158972024-02-08T06:45:32.105Z18/30 Write block 363 len 40960 data: 7 11 11 9 12 9 12 12 12 16
158982024-02-08T06:45:32.105Z19/30 Read block 49 len 24576
158992024-02-08T06:45:32.107Z20/30 Read block 282 len 28672
159002024-02-08T06:45:32.108Z21/30 Read block 530 len 24576
159012024-02-08T06:45:32.111Z22/30 Write block 392 len 40960 data: 10 8 8 7 5 8 10 11 11 13
159022024-02-08T06:45:32.111Z23/30 Write block 174 len 36864 data: 10 11 10 9 7 7 8 13 14
159032024-02-08T06:45:32.111Z24/30 Write block 539 len 12288 data: 9 9 9
159042024-02-08T06:45:32.111Z25/30 Write block 488 len 32768 data: 13 11 11 12 11 12 10 9
159052024-02-08T06:45:32.114Z26/30 Write block 364 len 12288 data: 12 12 10
159062024-02-08T06:45:32.114Z27/30 Write block 471 len 16384 data: 10 12 11 13
159072024-02-08T06:45:32.114Z28/30 Write block 42 len 8192 data: 8 5
159082024-02-08T06:45:32.115Z29/30 Read block 490 len 32768
159092024-02-08T06:45:32.115Z30/30 Read block 168 len 36864
159102024-02-08T06:45:32.118Z----------------------------------------------------------------
159112024-02-08T06:45:32.118Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
159122024-02-08T06:45:32.119ZStates: Active Active Active
159132024-02-08T06:45:32.119ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159142024-02-08T06:45:32.119Z 1 Acked 1000 Write 8 Done Done Done false
159152024-02-08T06:45:32.119Z 2 Acked 1001 Read 8 Done Done Done false
159162024-02-08T06:45:32.119Z 3 Acked 1002 Read 10 Done Sent Done false
159172024-02-08T06:45:32.119Z 4 Acked 1003 Flush 0 Done Sent Done false
159182024-02-08T06:45:32.119Z 5 Acked 1004 Write 1 Done Sent Done false
159192024-02-08T06:45:32.119Z 6 Acked 1005 Read 7 Done Sent Done false
159202024-02-08T06:45:32.119Z 7 Acked 1006 Read 5 Done Sent Done false
159212024-02-08T06:45:32.119Z 8 Acked 1007 Read 4 Done Sent Done false
159222024-02-08T06:45:32.119Z 9 Acked 1008 Read 8 Done Sent Done false
159232024-02-08T06:45:32.119Z 10 Acked 1009 Flush 0 Done Sent Done false
159242024-02-08T06:45:32.119Z 11 Acked 1010 Write 6 Done Sent Done false
159252024-02-08T06:45:32.119Z 12 Acked 1011 Flush 0 Done Sent Done false
159262024-02-08T06:45:32.119Z 13 Acked 1012 Write 8 Done Sent Done false
159272024-02-08T06:45:32.120Z 14 Acked 1013 Read 4 Done Sent Done false
159282024-02-08T06:45:32.120Z 15 Acked 1014 Write 7 Done Sent Done false
159292024-02-08T06:45:32.120Z 16 Acked 1015 Read 8 Done Sent Done false
159302024-02-08T06:45:32.120Z 17 Acked 1016 Write 8 Done Sent Done false
159312024-02-08T06:45:32.120Z 18 Acked 1017 Write 10 Done Sent Done false
159322024-02-08T06:45:32.120Z 19 Acked 1018 Read 6 Done Sent Done false
159332024-02-08T06:45:32.120Z 20 Acked 1019 Read 7 Done Sent Done false
159342024-02-08T06:45:32.120Z 21 Acked 1020 Read 6 Done Sent Done false
159352024-02-08T06:45:32.120Z 22 Acked 1021 Write 10 Done Sent Done false
159362024-02-08T06:45:32.120Z 23 Acked 1022 Write 9 Done Sent Done false
159372024-02-08T06:45:32.120Z 24 Acked 1023 Write 3 Done Sent Done false
159382024-02-08T06:45:32.120Z 25 Acked 1024 Write 8 Done Sent Done false
159392024-02-08T06:45:32.120Z 26 Acked 1025 Write 3 Done Sent Done false
159402024-02-08T06:45:32.120Z 27 Acked 1026 Write 4 Done Sent Done false
159412024-02-08T06:45:32.120Z 28 Acked 1027 Write 2 Done Sent Done false
159422024-02-08T06:45:32.121Z 29 Acked 1028 Read 8 Done Sent Done false
159432024-02-08T06:45:32.121Z 30 Acked 1029 Read 9 Sent Sent Done false
159442024-02-08T06:45:32.121Z STATES DS:0 DS:1 DS:2 TOTAL
159452024-02-08T06:45:32.121Z New 0 0 0 0
159462024-02-08T06:45:32.121Z Sent 1 28 0 29
159472024-02-08T06:45:32.121Z Done 29 2 30 61
159482024-02-08T06:45:32.121Z Skipped 0 0 0 0
159492024-02-08T06:45:32.121Z Error 0 0 0 0
159502024-02-08T06:45:32.121ZLast Flush: 1011 0 1011
159512024-02-08T06:45:32.121ZDownstairs last five completed:
159522024-02-08T06:45:32.121ZUpstairs last five completed: 30 29 28 27 26
159532024-02-08T06:45:32.121Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:31.335485071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1722}
159542024-02-08T06:45:32.121ZWrote out file "/tmp/test_repair_verify.data"
159552024-02-08T06:45:32.134Z
159562024-02-08T06:45:32.134ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
159572024-02-08T06:45:32.142ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159582024-02-08T06:45:32.142Z 0 000-019 143 143 143 240 240 240 F F F
159592024-02-08T06:45:32.142Z 1 020-039 138 138 138 233 233 233 F F F
159602024-02-08T06:45:32.142Z 2 040-059 138 138 138 233 233 233 T F T <---
159612024-02-08T06:45:32.146Z 3 060-079 144 144 144 245 245 245 F F F
159622024-02-08T06:45:32.146Z 4 080-099 144 144 144 241 241 241 F F F
159632024-02-08T06:45:32.146Z 5 100-119 144 144 144 243 243 243 F F F
159642024-02-08T06:45:32.146Z 6 120-139 144 144 144 244 244 244 F F F
159652024-02-08T06:45:32.146Z 7 140-159 145 145 145 246 246 246 F F F
159662024-02-08T06:45:32.146Z 8 160-179 145 145 145 246 246 246 T F T <---
159672024-02-08T06:45:32.146Z 9 180-199 142 142 142 239 239 239 T F T <---
159682024-02-08T06:45:32.146Z 10 200-219 143 143 143 240 240 240 T F T <---
159692024-02-08T06:45:32.146Z 11 220-239 141 141 141 238 238 238 F F F
159702024-02-08T06:45:32.146Z 12 240-259 141 141 141 238 238 238 F F F
159712024-02-08T06:45:32.146Z 13 260-279 136 136 136 228 228 228 F F F
159722024-02-08T06:45:32.147Z 14 280-299 144 144 144 242 242 242 F F F
159732024-02-08T06:45:32.147Z 15 300-319 133 133 133 224 224 224 F F F
159742024-02-08T06:45:32.147Z 16 320-339 143 143 143 240 240 240 F F F
159752024-02-08T06:45:32.147Z 17 340-359 146 143 146 248 240 248 F F F <---
159762024-02-08T06:45:32.147Z 18 360-379 138 138 138 233 233 233 T F T <---
159772024-02-08T06:45:32.147Z 19 380-399 139 139 139 235 235 235 T F T <---
159782024-02-08T06:45:32.147Z 20 400-419 143 143 143 240 240 240 T F T <---
159792024-02-08T06:45:32.147Z 21 420-439 143 143 143 240 240 240 F F F
159802024-02-08T06:45:32.147Z 22 440-459 146 143 146 249 240 249 F F F <---
159812024-02-08T06:45:32.147Z 23 460-479 141 141 141 238 238 238 T F T <---
159822024-02-08T06:45:32.147Z 24 480-499 145 145 145 246 246 246 T F T <---
159832024-02-08T06:45:32.147Z 25 500-519 142 142 142 239 239 239 F F F
159842024-02-08T06:45:32.147Z 26 520-539 136 136 136 228 228 228 T F T <---
159852024-02-08T06:45:32.147Z 27 540-559 146 145 146 247 246 247 T T T <---
159862024-02-08T06:45:32.147Z 28 560-579 145 145 145 246 246 246 F F F
159872024-02-08T06:45:32.148Z 29 580-599 133 133 133 224 224 224 F F F
159882024-02-08T06:45:32.148ZMax gen: 146, Max flush: 249
159892024-02-08T06:45:32.148ZError: Difference in extent metadata found!
159902024-02-08T06:45:32.148ZA Difference in extent metadata is expected here
159912024-02-08T06:45:32.148ZOn loop 73
159922024-02-08T06:45:32.148Z
159932024-02-08T06:45:32.154ZVerifying data now
159942024-02-08T06:45:36.796ZNo change after verify
159952024-02-08T06:45:36.796ZLoop: 73 Downstairs dump after verify (and repair):
159962024-02-08T06:45:36.808ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159972024-02-08T06:45:36.808Z 0 000-019 143 143 143 240 240 240 F F F
159982024-02-08T06:45:36.809Z 1 020-039 138 138 138 233 233 233 F F F
159992024-02-08T06:45:36.809Z 2 040-059 147 147 147 250 250 250 F F F
160002024-02-08T06:45:36.809Z 3 060-079 144 144 144 245 245 245 F F F
160012024-02-08T06:45:36.809Z 4 080-099 144 144 144 241 241 241 F F F
160022024-02-08T06:45:36.809Z 5 100-119 144 144 144 243 243 243 F F F
160032024-02-08T06:45:36.809Z 6 120-139 144 144 144 244 244 244 F F F
160042024-02-08T06:45:36.809Z 7 140-159 145 145 145 246 246 246 F F F
160052024-02-08T06:45:36.809Z 8 160-179 147 147 147 250 250 250 F F F
160062024-02-08T06:45:36.809Z 9 180-199 147 147 147 250 250 250 F F F
160072024-02-08T06:45:36.812Z 10 200-219 147 147 147 250 250 250 F F F
160082024-02-08T06:45:36.813Z 11 220-239 141 141 141 238 238 238 F F F
160092024-02-08T06:45:36.813Z 12 240-259 141 141 141 238 238 238 F F F
160102024-02-08T06:45:36.813Z 13 260-279 136 136 136 228 228 228 F F F
160112024-02-08T06:45:36.813Z 14 280-299 144 144 144 242 242 242 F F F
160122024-02-08T06:45:36.813Z 15 300-319 133 133 133 224 224 224 F F F
160132024-02-08T06:45:36.813Z 16 320-339 143 143 143 240 240 240 F F F
160142024-02-08T06:45:36.813Z 17 340-359 146 146 146 248 248 248 F F F
160152024-02-08T06:45:36.813Z 18 360-379 147 147 147 250 250 250 F F F
160162024-02-08T06:45:36.813Z 19 380-399 147 147 147 250 250 250 F F F
160172024-02-08T06:45:36.813Z 20 400-419 147 147 147 250 250 250 F F F
160182024-02-08T06:45:36.813Z 21 420-439 143 143 143 240 240 240 F F F
160192024-02-08T06:45:36.813Z 22 440-459 146 146 146 249 249 249 F F F
160202024-02-08T06:45:36.813Z 23 460-479 147 147 147 250 250 250 F F F
160212024-02-08T06:45:36.813Z 24 480-499 147 147 147 250 250 250 F F F
160222024-02-08T06:45:36.814Z 25 500-519 142 142 142 239 239 239 F F F
160232024-02-08T06:45:36.814Z 26 520-539 147 147 147 250 250 250 F F F
160242024-02-08T06:45:36.814Z 27 540-559 147 147 147 250 250 250 F F F
160252024-02-08T06:45:36.814Z 28 560-579 145 145 145 246 246 246 F F F
160262024-02-08T06:45:36.814Z 29 580-599 133 133 133 224 224 224 F F F
160272024-02-08T06:45:36.814ZMax gen: 147, Max flush: 250
160282024-02-08T06:45:36.814Z
160292024-02-08T06:45:36.814ZBegin loop 74
160302024-02-08T06:45:36.814ZDownstairs to restart: 1
160312024-02-08T06:45:36.830Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048135855Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730}
160322024-02-08T06:45:36.830Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
160332024-02-08T06:45:36.830Z,"time":"2024-02-08T06:45:36.048475129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730}
160342024-02-08T06:45:36.837Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048523746Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730}
160352024-02-08T06:45:36.837Z{"msg":"Crucible b984f906-4880-4b95-ae5f-ad2eee9e595b has session id: 86ac5ce9-bd4d-47a5-80e5-19b91cadd043","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048573072Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160362024-02-08T06:45:36.837Z{"msg":"Upstairs opts: Upstairs UUID: b984f906-4880-4b95-ae5f-ad2eee9e595b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048621858Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160372024-02-08T06:45:36.837Z{"msg":"Crucible stats registered with UUID: b984f906-4880-4b95-ae5f-ad2eee9e595b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048662645Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160382024-02-08T06:45:36.837Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048691993Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730}
160392024-02-08T06:45:36.837Z{"msg":"b984f906-4880-4b95-ae5f-ad2eee9e595b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.0487279Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160402024-02-08T06:45:36.837Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048757328Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160412024-02-08T06:45:36.837Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048804635Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160422024-02-08T06:45:36.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048849301Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160432024-02-08T06:45:36.838Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048882999Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160442024-02-08T06:45:36.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048937214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160452024-02-08T06:45:36.838Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.048970892Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160462024-02-08T06:45:36.838Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.049011569Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160472024-02-08T06:45:36.838Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.049051316Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160482024-02-08T06:45:36.838Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.049078984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160492024-02-08T06:45:36.838Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.049119231Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160502024-02-08T06:45:36.838Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.049168247Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160512024-02-08T06:45:36.838Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.049196245Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160522024-02-08T06:45:36.838Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.049724175Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160532024-02-08T06:45:36.838Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.049771972Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160542024-02-08T06:45:36.838Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.04979864Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160552024-02-08T06:45:36.838Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.049837667Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160562024-02-08T06:45:36.838Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.050364947Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160572024-02-08T06:45:36.838Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.050454791Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160582024-02-08T06:45:36.839Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.050931555Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160592024-02-08T06:45:36.839Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.050973522Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160602024-02-08T06:45:36.839Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.050989231Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160612024-02-08T06:45:36.839Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.051046746Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160622024-02-08T06:45:36.839Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.051076604Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160632024-02-08T06:45:36.839Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.051091313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160642024-02-08T06:45:36.839Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.051112081Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160652024-02-08T06:45:36.839Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.05113282Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160662024-02-08T06:45:36.839Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.051615013Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160672024-02-08T06:45:36.839Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052086938Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160682024-02-08T06:45:36.839Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052133115Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160692024-02-08T06:45:36.839Z{"msg":"[0]R flush_numbers[0..12]: [240, 233, 250, 245, 241, 243, 244, 246, 250, 250, 250, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052155233Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160702024-02-08T06:45:36.839Z{"msg":"[0]R generation[0..12]: [143, 138, 147, 144, 144, 144, 144, 145, 147, 147, 147, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052183691Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160712024-02-08T06:45:36.839Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052203969Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160722024-02-08T06:45:36.839Z{"msg":"[1]R flush_numbers[0..12]: [240, 233, 250, 245, 241, 243, 244, 246, 250, 250, 250, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052230897Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160732024-02-08T06:45:36.840ZWait for a query_work_queue command to finish before sending IO
160742024-02-08T06:45:36.840Z{"msg":"[1]R generation[0..12]: [143, 138, 147, 144, 144, 144, 144, 145, 147, 147, 147, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052254485Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160752024-02-08T06:45:36.840Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052282943Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160762024-02-08T06:45:36.840Z{"msg":"[2]R flush_numbers[0..12]: [240, 233, 250, 245, 241, 243, 244, 246, 250, 250, 250, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052302912Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160772024-02-08T06:45:36.840Z{"msg":"[2]R generation[0..12]: [143, 138, 147, 144, 144, 144, 144, 145, 147, 147, 147, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.05232994Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160782024-02-08T06:45:36.840Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052363077Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160792024-02-08T06:45:36.840Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052384636Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160802024-02-08T06:45:36.840Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052402884Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160812024-02-08T06:45:36.840Z{"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052420313Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160822024-02-08T06:45:36.840Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052442491Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160832024-02-08T06:45:36.840Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052481408Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160842024-02-08T06:45:36.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052517446Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160852024-02-08T06:45:36.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
160862024-02-08T06:45:36.840Z,"time":"2024-02-08T06:45:36.052558283Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160872024-02-08T06:45:36.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.05259847Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160882024-02-08T06:45:36.841Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052637037Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160892024-02-08T06:45:36.841Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052674874Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160902024-02-08T06:45:36.841Z{"msg":"b984f906-4880-4b95-ae5f-ad2eee9e595b is now active with session: 86ac5ce9-bd4d-47a5-80e5-19b91cadd043","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052712451Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160912024-02-08T06:45:36.841Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052749668Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730,"session_id":"86ac5ce9-bd4d-47a5-80e5-19b91cadd043"}
160922024-02-08T06:45:36.841Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:36.052771687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730}
160932024-02-08T06:45:36.841ZLoading write count information from file "/tmp/test_repair_verify.data"
160942024-02-08T06:45:36.841ZRun Repair workload
160952024-02-08T06:45:36.841Z01/30 Write block 265 len 20480 data: 14 14 15 15 12
160962024-02-08T06:45:36.844Z02/30 Read block 582 len 36864
160972024-02-08T06:45:36.845Z03/30 Read block 395 len 12288
160982024-02-08T06:45:36.845Z04/30 Read block 223 len 28672
160992024-02-08T06:45:36.845Z05/30 Write block 366 len 40960 data: 11 13 10 13 13 13 17 15 7 10
161002024-02-08T06:45:36.845Z06/30 Write block 234 len 36864 data: 9 11 9 8 10 11 10 13 10
161012024-02-08T06:45:36.845Z07/30 Read block 124 len 36864
161022024-02-08T06:45:36.845Z08/30 Read block 552 len 8192
161032024-02-08T06:45:36.845Z09/30 Flush
161042024-02-08T06:45:36.845Z10/30 Write block 243 len 40960 data: 14 11 11 11 9 10 14 14 14 11
161052024-02-08T06:45:36.848Z11/30 Write block 94 len 16384 data: 18 18 18 19
161062024-02-08T06:45:36.848Z12/30 Flush
161072024-02-08T06:45:36.851Z13/30 Write block 33 len 28672 data: 10 9 12 11 10 12 9
161082024-02-08T06:45:36.851Z14/30 Write block 238 len 36864 data: 11 12 11 14 11 15 12 12 12
161092024-02-08T06:45:36.851Z15/30 Write block 273 len 32768 data: 9 9 10 12 14 12 16 14
161102024-02-08T06:45:36.854Z16/30 Write block 535 len 36864 data: 14 14 12 11 10 10 10 8 10
161112024-02-08T06:45:36.854Z17/30 Read block 397 len 16384
161122024-02-08T06:45:36.854Z18/30 Read block 134 len 20480
161132024-02-08T06:45:36.854Z19/30 Read block 185 len 16384
161142024-02-08T06:45:36.857Z20/30 Read block 214 len 32768
161152024-02-08T06:45:36.857Z21/30 Read block 32 len 24576
161162024-02-08T06:45:36.857Z22/30 Read block 351 len 4096
161172024-02-08T06:45:36.860Z23/30 Write block 164 len 32768 data: 13 14 15 18 15 15 13 11
161182024-02-08T06:45:36.860Z24/30 Write block 494 len 4096 data: 11
161192024-02-08T06:45:36.860Z25/30 Flush
161202024-02-08T06:45:36.864Z26/30 Write block 170 len 4096 data: 14
161212024-02-08T06:45:36.864Z27/30 Write block 423 len 8192 data: 11 10
161222024-02-08T06:45:36.864Z28/30 Write block 197 len 16384 data: 13 16 17 15
161232024-02-08T06:45:36.864Z29/30 Read block 383 len 40960
161242024-02-08T06:45:36.868Z30/30 Write block 579 len 8192 data: 12 9
161252024-02-08T06:45:36.868Z----------------------------------------------------------------
161262024-02-08T06:45:36.868Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
161272024-02-08T06:45:36.868ZStates: Active Active Active
161282024-02-08T06:45:36.868ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
161292024-02-08T06:45:36.868Z 1 Acked 1000 Write 5 Done Done Done false
161302024-02-08T06:45:36.868Z 2 Acked 1001 Read 9 Done Done Done false
161312024-02-08T06:45:36.868Z 3 Acked 1002 Read 3 Done Sent Done false
161322024-02-08T06:45:36.868Z 4 Acked 1003 Read 7 Done Sent Done false
161332024-02-08T06:45:36.868Z 5 Acked 1004 Write 10 Done Sent Done false
161342024-02-08T06:45:36.868Z 6 Acked 1005 Write 9 Done Sent Done false
161352024-02-08T06:45:36.868Z 7 Acked 1006 Read 9 Done Sent Done false
161362024-02-08T06:45:36.868Z 8 Acked 1007 Read 2 Done Sent Done false
161372024-02-08T06:45:36.868Z 9 Acked 1008 Flush 0 Done Sent Done false
161382024-02-08T06:45:36.869Z 10 Acked 1009 Write 10 Done Sent Done false
161392024-02-08T06:45:36.869Z 11 Acked 1010 Write 4 Done Sent Done false
161402024-02-08T06:45:36.869Z 12 Acked 1011 Flush 0 Done Sent Done false
161412024-02-08T06:45:36.869Z 13 Acked 1012 Write 7 Done Sent Done false
161422024-02-08T06:45:36.869Z 14 Acked 1013 Write 9 Done Sent Done false
161432024-02-08T06:45:36.869Z 15 Acked 1014 Write 8 Done Sent Done false
161442024-02-08T06:45:36.869Z 16 Acked 1015 Write 9 Done Sent Done false
161452024-02-08T06:45:36.869Z 17 Acked 1016 Read 4 Done Sent Done false
161462024-02-08T06:45:36.869Z 18 Acked 1017 Read 5 Done Sent Done false
161472024-02-08T06:45:36.869Z 19 Acked 1018 Read 4 Done Sent Done false
161482024-02-08T06:45:36.869Z 20 Acked 1019 Read 8 Done Sent Done false
161492024-02-08T06:45:36.869Z 21 Acked 1020 Read 6 Done Sent Done false
161502024-02-08T06:45:36.869Z 22 Acked 1021 Read 1 Done Sent Done false
161512024-02-08T06:45:36.869Z 23 Acked 1022 Write 8 Done Sent Done false
161522024-02-08T06:45:36.869Z 24 Acked 1023 Write 1 Done Sent Done false
161532024-02-08T06:45:36.870Z 25 Acked 1024 Flush 0 Done Sent Done false
161542024-02-08T06:45:36.870Z 26 Acked 1025 Write 1 Done Sent Done false
161552024-02-08T06:45:36.870Z 27 Acked 1026 Write 2 Done Sent Done false
161562024-02-08T06:45:36.870Z 28 Acked 1027 Write 4 Done Sent Done false
161572024-02-08T06:45:36.870Z 29 Acked 1028 Read 10 Sent Sent Done false
161582024-02-08T06:45:36.870Z 30 Acked 1029 Write 2 Sent Sent Sent false
161592024-02-08T06:45:36.870Z STATES DS:0 DS:1 DS:2 TOTAL
161602024-02-08T06:45:36.870Z New 0 0 0 0
161612024-02-08T06:45:36.870Z Sent 2 28 1 31
161622024-02-08T06:45:36.870Z Done 28 2 29 59
161632024-02-08T06:45:36.870Z Skipped 0 0 0 0
161642024-02-08T06:45:36.870Z Error 0 0 0 0
161652024-02-08T06:45:36.870ZLast Flush: 1024 0 1024
161662024-02-08T06:45:36.870ZDownstairs last five completed:
161672024-02-08T06:45:36.870ZUpstairs last five completed: 30 29 28 27 26
161682024-02-08T06:45:36.871Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:36.0856227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1730}
161692024-02-08T06:45:36.871ZWrote out file "/tmp/test_repair_verify.data"
161702024-02-08T06:45:36.875Z
161712024-02-08T06:45:36.875ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
161722024-02-08T06:45:36.891ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161732024-02-08T06:45:36.891Z 0 000-019 143 143 143 240 240 240 F F F
161742024-02-08T06:45:36.891Z 1 020-039 148 138 148 253 233 253 F F F <---
161752024-02-08T06:45:36.891Z 2 040-059 147 147 147 250 250 250 F F F
161762024-02-08T06:45:36.891Z 3 060-079 144 144 144 245 245 245 F F F
161772024-02-08T06:45:36.891Z 4 080-099 148 144 148 252 241 252 F F F <---
161782024-02-08T06:45:36.891Z 5 100-119 144 144 144 243 243 243 F F F
161792024-02-08T06:45:36.891Z 6 120-139 144 144 144 244 244 244 F F F
161802024-02-08T06:45:36.895Z 7 140-159 145 145 145 246 246 246 F F F
161812024-02-08T06:45:36.895Z 8 160-179 148 147 148 253 250 253 T F T <---
161822024-02-08T06:45:36.895Z 9 180-199 147 147 147 250 250 250 T F T <---
161832024-02-08T06:45:36.895Z 10 200-219 147 147 147 250 250 250 T F T <---
161842024-02-08T06:45:36.896Z 11 220-239 148 141 148 253 238 253 F F F <---
161852024-02-08T06:45:36.896Z 12 240-259 148 141 148 253 238 253 F F F <---
161862024-02-08T06:45:36.896Z 13 260-279 148 136 148 253 228 253 F T F <---
161872024-02-08T06:45:36.896Z 14 280-299 148 144 148 253 242 253 F F F <---
161882024-02-08T06:45:36.896Z 15 300-319 133 133 133 224 224 224 F F F
161892024-02-08T06:45:36.896Z 16 320-339 143 143 143 240 240 240 F F F
161902024-02-08T06:45:36.896Z 17 340-359 146 146 146 248 248 248 F F F
161912024-02-08T06:45:36.896Z 18 360-379 148 147 148 251 250 251 F F F <---
161922024-02-08T06:45:36.896Z 19 380-399 147 147 147 250 250 250 F F F
161932024-02-08T06:45:36.896Z 20 400-419 147 147 147 250 250 250 F F F
161942024-02-08T06:45:36.896Z 21 420-439 143 143 143 240 240 240 T F T <---
161952024-02-08T06:45:36.896Z 22 440-459 146 146 146 249 249 249 F F F
161962024-02-08T06:45:36.896Z 23 460-479 147 147 147 250 250 250 F F F
161972024-02-08T06:45:36.896Z 24 480-499 148 147 148 253 250 253 F F F <---
161982024-02-08T06:45:36.896Z 25 500-519 142 142 142 239 239 239 F F F
161992024-02-08T06:45:36.897Z 26 520-539 148 147 148 253 250 253 F F F <---
162002024-02-08T06:45:36.897Z 27 540-559 148 147 148 253 250 253 F F F <---
162012024-02-08T06:45:36.897Z 28 560-579 145 145 145 246 246 246 T F T <---
162022024-02-08T06:45:36.897Z 29 580-599 133 133 133 224 224 224 T F T <---
162032024-02-08T06:45:36.897ZMax gen: 148, Max flush: 253
162042024-02-08T06:45:36.897ZError: Difference in extent metadata found!
162052024-02-08T06:45:36.897ZA Difference in extent metadata is expected here
162062024-02-08T06:45:36.897ZOn loop 74
162072024-02-08T06:45:36.897Z
162082024-02-08T06:45:36.901ZVerifying data now
162092024-02-08T06:45:41.657ZNo change after verify
162102024-02-08T06:45:41.657ZLoop: 74 Downstairs dump after verify (and repair):
162112024-02-08T06:45:41.672ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162122024-02-08T06:45:41.672Z 0 000-019 143 143 143 240 240 240 F F F
162132024-02-08T06:45:41.672Z 1 020-039 148 148 148 253 253 253 F F F
162142024-02-08T06:45:41.672Z 2 040-059 147 147 147 250 250 250 F F F
162152024-02-08T06:45:41.672Z 3 060-079 144 144 144 245 245 245 F F F
162162024-02-08T06:45:41.672Z 4 080-099 148 148 148 252 252 252 F F F
162172024-02-08T06:45:41.672Z 5 100-119 144 144 144 243 243 243 F F F
162182024-02-08T06:45:41.676Z 6 120-139 144 144 144 244 244 244 F F F
162192024-02-08T06:45:41.676Z 7 140-159 145 145 145 246 246 246 F F F
162202024-02-08T06:45:41.676Z 8 160-179 149 149 149 254 254 254 F F F
162212024-02-08T06:45:41.676Z 9 180-199 149 149 149 254 254 254 F F F
162222024-02-08T06:45:41.676Z 10 200-219 149 149 149 254 254 254 F F F
162232024-02-08T06:45:41.676Z 11 220-239 148 148 148 253 253 253 F F F
162242024-02-08T06:45:41.676Z 12 240-259 148 148 148 253 253 253 F F F
162252024-02-08T06:45:41.676Z 13 260-279 148 148 148 253 253 253 F F F
162262024-02-08T06:45:41.676Z 14 280-299 148 148 148 253 253 253 F F F
162272024-02-08T06:45:41.676Z 15 300-319 133 133 133 224 224 224 F F F
162282024-02-08T06:45:41.677Z 16 320-339 143 143 143 240 240 240 F F F
162292024-02-08T06:45:41.677Z 17 340-359 146 146 146 248 248 248 F F F
162302024-02-08T06:45:41.677Z 18 360-379 148 148 148 251 251 251 F F F
162312024-02-08T06:45:41.677Z 19 380-399 147 147 147 250 250 250 F F F
162322024-02-08T06:45:41.677Z 20 400-419 147 147 147 250 250 250 F F F
162332024-02-08T06:45:41.677Z 21 420-439 149 149 149 254 254 254 F F F
162342024-02-08T06:45:41.677Z 22 440-459 146 146 146 249 249 249 F F F
162352024-02-08T06:45:41.677Z 23 460-479 147 147 147 250 250 250 F F F
162362024-02-08T06:45:41.677Z 24 480-499 148 148 148 253 253 253 F F F
162372024-02-08T06:45:41.677Z 25 500-519 142 142 142 239 239 239 F F F
162382024-02-08T06:45:41.677Z 26 520-539 148 148 148 253 253 253 F F F
162392024-02-08T06:45:41.677Z 27 540-559 148 148 148 253 253 253 F F F
162402024-02-08T06:45:41.677Z 28 560-579 149 149 149 254 254 254 F F F
162412024-02-08T06:45:41.677Z 29 580-599 149 149 149 254 254 254 F F F
162422024-02-08T06:45:41.678ZMax gen: 149, Max flush: 254
162432024-02-08T06:45:41.678Z
162442024-02-08T06:45:41.678ZBegin loop 75
162452024-02-08T06:45:41.678ZDownstairs to restart: 1
162462024-02-08T06:45:41.692Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.911573148Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738}
162472024-02-08T06:45:41.698Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.911843018Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal"Crucible runtime is spawned
162482024-02-08T06:45:41.698Z,"pid":1738}
162492024-02-08T06:45:41.698Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.911880745Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738}
162502024-02-08T06:45:41.698Z{"msg":"Crucible b7bbcea4-e101-4fcc-b745-e65173792426 has session id: c8471bb6-8072-4e8d-b898-9b33780b3432","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.911914603Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162512024-02-08T06:45:41.698Z{"msg":"Upstairs opts: Upstairs UUID: b7bbcea4-e101-4fcc-b745-e65173792426, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.91194788Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162522024-02-08T06:45:41.698Z{"msg":"Crucible stats registered with UUID: b7bbcea4-e101-4fcc-b745-e65173792426","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.911997486Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162532024-02-08T06:45:41.698Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912022915Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738}
162542024-02-08T06:45:41.698Z{"msg":"b7bbcea4-e101-4fcc-b745-e65173792426 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912062752Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162552024-02-08T06:45:41.698Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.91208663Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162562024-02-08T06:45:41.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912122657Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162572024-02-08T06:45:41.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912148825Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162582024-02-08T06:45:41.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912182892Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162592024-02-08T06:45:41.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912209241Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162602024-02-08T06:45:41.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912232789Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162612024-02-08T06:45:41.699Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912257207Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162622024-02-08T06:45:41.699Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912284095Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162632024-02-08T06:45:41.699Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912321572Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162642024-02-08T06:45:41.699Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912358939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162652024-02-08T06:45:41.699Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912398116Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162662024-02-08T06:45:41.699Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.912512328Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162672024-02-08T06:45:41.699Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.913134351Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162682024-02-08T06:45:41.699Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.913185577Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162692024-02-08T06:45:41.699Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.913218865Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162702024-02-08T06:45:41.699Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.913257682Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162712024-02-08T06:45:41.700Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.913777543Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162722024-02-08T06:45:41.700Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.9139416Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162732024-02-08T06:45:41.700Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.914241548Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162742024-02-08T06:45:41.700Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.914285185Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162752024-02-08T06:45:41.700Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.914311263Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162762024-02-08T06:45:41.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.914332221Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162772024-02-08T06:45:41.700Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.914356479Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162782024-02-08T06:45:41.700Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.914451512Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162792024-02-08T06:45:41.700Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.914489869Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162802024-02-08T06:45:41.700Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.914521557Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162812024-02-08T06:45:41.700Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.914854242Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162822024-02-08T06:45:41.700Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915272811Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162832024-02-08T06:45:41.700Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915313397Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162842024-02-08T06:45:41.706Z{"msg":"[0]R flush_numbers[0..12]: [240, 253, 250, 245, 252, 243, 244, 246, 254, 254, 254, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915330836Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162852024-02-08T06:45:41.706Z{"msg":"[0]R generation[0..12]: [143, 148, 147, 144, 148, 144, 144, 145, 149, 149, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915402511Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162862024-02-08T06:45:41.706Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915439608Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162872024-02-08T06:45:41.706Z{"msg":"[1]R flush_numbers[0..12]: [240, 253, 250, 245, 252, 243, 244, 246, 254, 254, 254, 253]","v":0,"Wait for a query_work_queue command to finish before sending IO
162882024-02-08T06:45:41.706Zname":"crucible","level":30,"time":"2024-02-08T06:45:40.915471426Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162892024-02-08T06:45:41.706Z{"msg":"[1]R generation[0..12]: [143, 148, 147, 144, 148, 144, 144, 145, 149, 149, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915496924Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162902024-02-08T06:45:41.706Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915535051Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162912024-02-08T06:45:41.706Z{"msg":"[2]R flush_numbers[0..12]: [240, 253, 250, 245, 252, 243, 244, 246, 254, 254, 254, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915572538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162922024-02-08T06:45:41.706Z{"msg":"[2]R generation[0..12]: [143, 148, 147, 144, 148, 144, 144, 145, 149, 149, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915602106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162932024-02-08T06:45:41.706Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915617575Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162942024-02-08T06:45:41.706Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915639183Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162952024-02-08T06:45:41.706Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915668151Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162962024-02-08T06:45:41.707Z{"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.91568271Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162972024-02-08T06:45:41.707Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915714727Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162982024-02-08T06:45:41.707Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915731166Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
162992024-02-08T06:45:41.707Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915759314Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
163002024-02-08T06:45:41.707Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915775883Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
163012024-02-08T06:45:41.707Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915805541Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
163022024-02-08T06:45:41.707Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915820359Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
163032024-02-08T06:45:41.707Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915850197Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
163042024-02-08T06:45:41.707Z{"msg":"b7bbcea4-e101-4fcc-b745-e65173792426 is now active with session: c8471bb6-8072-4e8d-b898-9b33780b3432","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915866356Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
163052024-02-08T06:45:41.707Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163062024-02-08T06:45:41.707Z,"time":"2024-02-08T06:45:40.915894434Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738,"session_id":"c8471bb6-8072-4e8d-b898-9b33780b3432"}
163072024-02-08T06:45:41.707Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:40.915909433Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738}
163082024-02-08T06:45:41.707ZLoading write count information from file "/tmp/test_repair_verify.data"
163092024-02-08T06:45:41.707ZRun Repair workload
163102024-02-08T06:45:41.708Z01/30 Write block 98 len 36864 data: 17 16 17 15 13 13 14 11 10
163112024-02-08T06:45:41.708Z02/30 Read block 80 len 36864
163122024-02-08T06:45:41.708Z03/30 Write block 79 len 32768 data: 11 12 11 12 11 11 11 11
163132024-02-08T06:45:41.708Z04/30 Read block 474 len 40960
163142024-02-08T06:45:41.708Z05/30 Write block 143 len 20480 data: 7 6 6 5 7
163152024-02-08T06:45:41.708Z06/30 Write block 99 len 8192 data: 17 18
163162024-02-08T06:45:41.711Z07/30 Read block 591 len 32768
163172024-02-08T06:45:41.711Z08/30 Write block 275 len 20480 data: 11 13 15 13 17
163182024-02-08T06:45:41.711Z09/30 Read block 160 len 16384
163192024-02-08T06:45:41.711Z10/30 Read block 142 len 8192
163202024-02-08T06:45:41.711Z11/30 Write block 591 len 16384 data: 12 10 11 10
163212024-02-08T06:45:41.711Z12/30 Write block 365 len 16384 data: 13 12 14 11
163222024-02-08T06:45:41.712Z13/30 Write block 50 len 12288 data: 8 9 8
163232024-02-08T06:45:41.712Z14/30 Read block 350 len 12288
163242024-02-08T06:45:41.712Z15/30 Flush
163252024-02-08T06:45:41.715Z16/30 Write block 475 len 4096 data: 12
163262024-02-08T06:45:41.715Z17/30 Write block 151 len 36864 data: 11 10 12 14 15 15 12 13 12
163272024-02-08T06:45:41.715Z18/30 Read block 529 len 32768
163282024-02-08T06:45:41.718Z19/30 Write block 512 len 16384 data: 7 9 7 6
163292024-02-08T06:45:41.718Z20/30 Write block 405 len 32768 data: 13 12 11 11 12 10 9 10
163302024-02-08T06:45:41.718Z21/30 Write block 46 len 8192 data: 8 6
163312024-02-08T06:45:41.718Z22/30 Read block 71 len 8192
163322024-02-08T06:45:41.718Z23/30 Flush
163332024-02-08T06:45:41.722Z24/30 Write block 159 len 36864 data: 13 11 12 14 14 14 15 16 19
163342024-02-08T06:45:41.722Z25/30 Read block 163 len 32768
163352024-02-08T06:45:41.725Z26/30 Read block 26 len 32768
163362024-02-08T06:45:41.725Z27/30 Read block 374 len 28672
163372024-02-08T06:45:41.725Z28/30 Read block 416 len 16384
163382024-02-08T06:45:41.728Z29/30 Read block 182 len 16384
163392024-02-08T06:45:41.729Z30/30 Write block 165 len 12288 data: 16 17 20
163402024-02-08T06:45:41.729Z----------------------------------------------------------------
163412024-02-08T06:45:41.729Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
163422024-02-08T06:45:41.729ZStates: Active Active Active
163432024-02-08T06:45:41.729ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
163442024-02-08T06:45:41.729Z 1 Acked 1000 Write 9 Done Done Done false
163452024-02-08T06:45:41.729Z 2 Acked 1001 Read 9 Done Done Done false
163462024-02-08T06:45:41.729Z 3 Acked 1002 Write 8 Done Done Done false
163472024-02-08T06:45:41.729Z 4 Acked 1003 Read 10 Done Done Done false
163482024-02-08T06:45:41.729Z 5 Acked 1004 Write 5 Done Sent Done false
163492024-02-08T06:45:41.729Z 6 Acked 1005 Write 2 Done Sent Done false
163502024-02-08T06:45:41.729Z 7 Acked 1006 Read 8 Done Sent Done false
163512024-02-08T06:45:41.729Z 8 Acked 1007 Write 5 Done Sent Done false
163522024-02-08T06:45:41.729Z 9 Acked 1008 Read 4 Done Sent Done false
163532024-02-08T06:45:41.730Z 10 Acked 1009 Read 2 Done Sent Done false
163542024-02-08T06:45:41.730Z 11 Acked 1010 Write 4 Done Sent Done false
163552024-02-08T06:45:41.730Z 12 Acked 1011 Write 4 Done Sent Done false
163562024-02-08T06:45:41.730Z 13 Acked 1012 Write 3 Done Sent Done false
163572024-02-08T06:45:41.730Z 14 Acked 1013 Read 3 Done Sent Done false
163582024-02-08T06:45:41.730Z 15 Acked 1014 Flush 0 Done Sent Done false
163592024-02-08T06:45:41.730Z 16 Acked 1015 Write 1 Done Sent Done false
163602024-02-08T06:45:41.730Z 17 Acked 1016 Write 9 Done Sent Done false
163612024-02-08T06:45:41.730Z 18 Acked 1017 Read 8 Done Sent Done false
163622024-02-08T06:45:41.730Z 19 Acked 1018 Write 4 Done Sent Done false
163632024-02-08T06:45:41.730Z 20 Acked 1019 Write 8 Done Sent Done false
163642024-02-08T06:45:41.730Z 21 Acked 1020 Write 2 Done Sent Done false
163652024-02-08T06:45:41.730Z 22 Acked 1021 Read 2 Done Sent Done false
163662024-02-08T06:45:41.730Z 23 Acked 1022 Flush 0 Done Sent Done false
163672024-02-08T06:45:41.730Z 24 Acked 1023 Write 9 Done Sent Done false
163682024-02-08T06:45:41.731Z 25 Acked 1024 Read 8 Done Sent Done false
163692024-02-08T06:45:41.731Z 26 Acked 1025 Read 8 Done Sent Done false
163702024-02-08T06:45:41.731Z 27 Acked 1026 Read 7 Done Sent Done false
163712024-02-08T06:45:41.731Z 28 Acked 1027 Read 4 Done Sent Done false
163722024-02-08T06:45:41.731Z 29 Acked 1028 Read 4 Done Sent Done false
163732024-02-08T06:45:41.731Z 30 Acked 1029 Write 3 Sent Sent Sent false
163742024-02-08T06:45:41.731Z STATES DS:0 DS:1 DS:2 TOTAL
163752024-02-08T06:45:41.731Z New 0 0 0 0
163762024-02-08T06:45:41.731Z Sent 1 26 1 28
163772024-02-08T06:45:41.731Z Done 29 4 29 62
163782024-02-08T06:45:41.731Z Skipped 0 0 0 0
163792024-02-08T06:45:41.731Z Error 0 0 0 0
163802024-02-08T06:45:41.731ZLast Flush: 1022 0 1022
163812024-02-08T06:45:41.731ZDownstairs last five completed:
163822024-02-08T06:45:41.731ZUpstairs last five completed: 30 29 28 27 26
163832024-02-08T06:45:41.731Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:40.946337028Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1738}
163842024-02-08T06:45:41.732ZWrote out file "/tmp/test_repair_verify.data"
163852024-02-08T06:45:41.736Z
163862024-02-08T06:45:41.736ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
163872024-02-08T06:45:41.750ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163882024-02-08T06:45:41.750Z 0 000-019 143 143 143 240 240 240 F F F
163892024-02-08T06:45:41.750Z 1 020-039 148 148 148 253 253 253 F F F
163902024-02-08T06:45:41.751Z 2 040-059 150 147 150 256 250 256 F F F <---
163912024-02-08T06:45:41.751Z 3 060-079 150 144 150 255 245 255 F T F <---
163922024-02-08T06:45:41.751Z 4 080-099 150 148 150 255 252 255 F T F <---
163932024-02-08T06:45:41.751Z 5 100-119 150 144 150 255 243 255 F T F <---
163942024-02-08T06:45:41.751Z 6 120-139 144 144 144 244 244 244 F F F
163952024-02-08T06:45:41.751Z 7 140-159 150 145 150 256 246 256 T F T <---
163962024-02-08T06:45:41.751Z 8 160-179 149 149 149 254 254 254 T F T <---
163972024-02-08T06:45:41.777Z 9 180-199 149 149 149 254 254 254 F F F
163982024-02-08T06:45:41.777Z 10 200-219 149 149 149 254 254 254 F F F
163992024-02-08T06:45:41.777Z 11 220-239 148 148 148 253 253 253 F F F
164002024-02-08T06:45:41.777Z 12 240-259 148 148 148 253 253 253 F F F
164012024-02-08T06:45:41.777Z 13 260-279 150 148 150 255 253 255 F F F <---
164022024-02-08T06:45:41.778Z 14 280-299 148 148 148 253 253 253 F F F
164032024-02-08T06:45:41.778Z 15 300-319 133 133 133 224 224 224 F F F
164042024-02-08T06:45:41.778Z 16 320-339 143 143 143 240 240 240 F F F
164052024-02-08T06:45:41.778Z 17 340-359 146 146 146 248 248 248 F F F
164062024-02-08T06:45:41.778Z 18 360-379 150 148 150 255 251 255 F F F <---
164072024-02-08T06:45:41.779Z 19 380-399 147 147 147 250 250 250 F F F
164082024-02-08T06:45:41.779Z 20 400-419 150 147 150 256 250 256 F F F <---
164092024-02-08T06:45:41.779Z 21 420-439 149 149 149 254 254 254 F F F
164102024-02-08T06:45:41.779Z 22 440-459 146 146 146 249 249 249 F F F
164112024-02-08T06:45:41.779Z 23 460-479 150 147 150 256 250 256 F F F <---
164122024-02-08T06:45:41.779Z 24 480-499 148 148 148 253 253 253 F F F
164132024-02-08T06:45:41.779Z 25 500-519 150 142 150 256 239 256 F F F <---
164142024-02-08T06:45:41.779Z 26 520-539 148 148 148 253 253 253 F F F
164152024-02-08T06:45:41.779Z 27 540-559 148 148 148 253 253 253 F F F
164162024-02-08T06:45:41.780Z 28 560-579 149 149 149 254 254 254 F F F
164172024-02-08T06:45:41.780Z 29 580-599 150 149 150 255 254 255 F F F <---
164182024-02-08T06:45:41.780ZMax gen: 150, Max flush: 256
164192024-02-08T06:45:41.780ZError: Difference in extent metadata found!
164202024-02-08T06:45:41.780ZA Difference in extent metadata is expected here
164212024-02-08T06:45:41.780ZOn loop 75
164222024-02-08T06:45:41.780Z
164232024-02-08T06:45:41.802ZVerifying data now
164242024-02-08T06:45:47.350ZNo change after verify
164252024-02-08T06:45:47.351ZLoop: 75 Downstairs dump after verify (and repair):
164262024-02-08T06:45:47.365ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164272024-02-08T06:45:47.365Z 0 000-019 143 143 143 240 240 240 F F F
164282024-02-08T06:45:47.366Z 1 020-039 148 148 148 253 253 253 F F F
164292024-02-08T06:45:47.366Z 2 040-059 150 150 150 256 256 256 F F F
164302024-02-08T06:45:47.366Z 3 060-079 150 150 150 255 255 255 F F F
164312024-02-08T06:45:47.366Z 4 080-099 150 150 150 255 255 255 F F F
164322024-02-08T06:45:47.369Z 5 100-119 150 150 150 255 255 255 F F F
164332024-02-08T06:45:47.369Z 6 120-139 144 144 144 244 244 244 F F F
164342024-02-08T06:45:47.369Z 7 140-159 151 151 151 257 257 257 F F F
164352024-02-08T06:45:47.369Z 8 160-179 151 151 151 257 257 257 F F F
164362024-02-08T06:45:47.369Z 9 180-199 149 149 149 254 254 254 F F F
164372024-02-08T06:45:47.369Z 10 200-219 149 149 149 254 254 254 F F F
164382024-02-08T06:45:47.370Z 11 220-239 148 148 148 253 253 253 F F F
164392024-02-08T06:45:47.370Z 12 240-259 148 148 148 253 253 253 F F F
164402024-02-08T06:45:47.370Z 13 260-279 150 150 150 255 255 255 F F F
164412024-02-08T06:45:47.370Z 14 280-299 148 148 148 253 253 253 F F F
164422024-02-08T06:45:47.370Z 15 300-319 133 133 133 224 224 224 F F F
164432024-02-08T06:45:47.370Z 16 320-339 143 143 143 240 240 240 F F F
164442024-02-08T06:45:47.370Z 17 340-359 146 146 146 248 248 248 F F F
164452024-02-08T06:45:47.370Z 18 360-379 150 150 150 255 255 255 F F F
164462024-02-08T06:45:47.370Z 19 380-399 147 147 147 250 250 250 F F F
164472024-02-08T06:45:47.370Z 20 400-419 150 150 150 256 256 256 F F F
164482024-02-08T06:45:47.370Z 21 420-439 149 149 149 254 254 254 F F F
164492024-02-08T06:45:47.370Z 22 440-459 146 146 146 249 249 249 F F F
164502024-02-08T06:45:47.371Z 23 460-479 150 150 150 256 256 256 F F F
164512024-02-08T06:45:47.371Z 24 480-499 148 148 148 253 253 253 F F F
164522024-02-08T06:45:47.371Z 25 500-519 150 150 150 256 256 256 F F F
164532024-02-08T06:45:47.371Z 26 520-539 148 148 148 253 253 253 F F F
164542024-02-08T06:45:47.371Z 27 540-559 148 148 148 253 253 253 F F F
164552024-02-08T06:45:47.371Z 28 560-579 149 149 149 254 254 254 F F F
164562024-02-08T06:45:47.371Z 29 580-599 150 150 150 255 255 255 F F F
164572024-02-08T06:45:47.371ZMax gen: 151, Max flush: 257
164582024-02-08T06:45:47.371Z
164592024-02-08T06:45:47.371ZBegin loop 76
164602024-02-08T06:45:47.371ZDownstairs to restart: 2
164612024-02-08T06:45:47.387Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.605238482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746}
164622024-02-08T06:45:47.387Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.60552575Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746}
164632024-02-08T06:45:47.463Z{"msg":"Crucible runtime is spawned
164642024-02-08T06:45:47.463ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.605624933Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746}
164652024-02-08T06:45:47.463Z{"msg":"Crucible 2530ab01-1201-43e8-aefb-3fc024e76db1 has session id: 8c40deaf-98df-45ea-bc00-0d8211a42bb7","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.6056621Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164662024-02-08T06:45:47.463Z{"msg":"Upstairs opts: Upstairs UUID: 2530ab01-1201-43e8-aefb-3fc024e76db1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.605709496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164672024-02-08T06:45:47.463Z{"msg":"Crucible stats registered with UUID: 2530ab01-1201-43e8-aefb-3fc024e76db1","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.605734614Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164682024-02-08T06:45:47.463Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.605772442Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746}
164692024-02-08T06:45:47.463Z{"msg":"2530ab01-1201-43e8-aefb-3fc024e76db1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.6057944Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164702024-02-08T06:45:47.463Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.605819518Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164712024-02-08T06:45:47.463Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.605876944Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164722024-02-08T06:45:47.463Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.6059258Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164732024-02-08T06:45:47.464Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.605970587Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164742024-02-08T06:45:47.464Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606002784Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164752024-02-08T06:45:47.464Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.60605925Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164762024-02-08T06:45:47.464Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606093298Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164772024-02-08T06:45:47.464Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606125555Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"io task","client":"0","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164782024-02-08T06:45:47.464Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606174192Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"io task","client":"1","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164792024-02-08T06:45:47.464Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:46.606231957Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164802024-02-08T06:45:47.464Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606278724Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"io task","client":"1","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164812024-02-08T06:45:47.464Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:46.606315601Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164822024-02-08T06:45:47.464Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:46.606370037Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164832024-02-08T06:45:47.464Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606388236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164842024-02-08T06:45:47.464Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606426953Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164852024-02-08T06:45:47.464Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.6064543Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164862024-02-08T06:45:47.464Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606490168Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"io task","client":"0","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164872024-02-08T06:45:47.464Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606703612Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164882024-02-08T06:45:47.465Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606751078Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164892024-02-08T06:45:47.465Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606795185Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164902024-02-08T06:45:47.465Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.606827283Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164912024-02-08T06:45:47.465Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.607377391Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164922024-02-08T06:45:47.465Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.607415068Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164932024-02-08T06:45:47.465Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.607955598Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164942024-02-08T06:45:47.465Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.608013554Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164952024-02-08T06:45:47.465Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.6080563Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164962024-02-08T06:45:47.465Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.608076769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164972024-02-08T06:45:47.465Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.608106587Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164982024-02-08T06:45:47.465Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:46.608135974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
164992024-02-08T06:45:48.403Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.607941771Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165002024-02-08T06:45:48.408Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.608167265Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165012024-02-08T06:45:48.408Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.608939547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165022024-02-08T06:45:48.408Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.608978044Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165032024-02-08T06:45:48.408Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.609454228Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165042024-02-08T06:45:48.408Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610004487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165052024-02-08T06:45:48.408Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610050503Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165062024-02-08T06:45:48.408Z{"msg":"[0]R flush_numbers[0..12]: [240, 253, 256, 255, 255, 255, 244, 257, 257, 254, 254, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.61008608Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165072024-02-08T06:45:48.408Z{"msg":"[0]R generation[0..12]: [143, 148, 150, 150, 150, 150, 144, 151, 151, 149, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610122368Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165082024-02-08T06:45:48.408Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610159085Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165092024-02-08T06:45:48.408Z{"msg":"[1]R flush_numbers[0..12]: [240, 253, 256, 255, 255, 255, 244, 257, 257, 254, 254, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610194732Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165102024-02-08T06:45:48.408Z{"msg":"[1]R generation[0..12]: [143, 148, 150, 150, 150, 150, 144, 151, 151, 149, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.61022915Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165112024-02-08T06:45:48.408Z{"msg":"Wait for a query_work_queue command to finish before sending IO
165122024-02-08T06:45:48.408Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610277426Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165132024-02-08T06:45:48.408Z{"msg":"[2]R flush_numbers[0..12]: [240, 253, 256, 255, 255, 255, 244, 257, 257, 254, 254, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610309134Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165142024-02-08T06:45:48.409Z{"msg":"[2]R generation[0..12]: [143, 148, 150, 150, 150, 150, 144, 151, 151, 149, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610348131Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165152024-02-08T06:45:48.409Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610388718Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165162024-02-08T06:45:48.409Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610428055Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165172024-02-08T06:45:48.409Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610464312Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165182024-02-08T06:45:48.409Z{"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610498769Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165192024-02-08T06:45:48.409Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610536716Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165202024-02-08T06:45:48.409Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610572074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165212024-02-08T06:45:48.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610600632Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165222024-02-08T06:45:48.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165232024-02-08T06:45:48.409Z,"time":"2024-02-08T06:45:47.610638229Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165242024-02-08T06:45:48.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610682576Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165252024-02-08T06:45:48.409Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610715423Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165262024-02-08T06:45:48.409Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610746191Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165272024-02-08T06:45:48.410Z{"msg":"2530ab01-1201-43e8-aefb-3fc024e76db1 is now active with session: 8c40deaf-98df-45ea-bc00-0d8211a42bb7","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610779658Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165282024-02-08T06:45:48.410Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610810846Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746,"session_id":"8c40deaf-98df-45ea-bc00-0d8211a42bb7"}
165292024-02-08T06:45:48.410Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:47.610841014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746}
165302024-02-08T06:45:48.410ZLoading write count information from file "/tmp/test_repair_verify.data"
165312024-02-08T06:45:48.410ZRun Repair workload
165322024-02-08T06:45:48.410Z01/30 Write block 211 len 32768 data: 17 19 17 15 13 13 14 13
165332024-02-08T06:45:48.410Z02/30 Write block 490 len 16384 data: 12 13 12 13
165342024-02-08T06:45:48.410Z03/30 Read block 469 len 28672
165352024-02-08T06:45:48.410Z04/30 Write block 29 len 32768 data: 12 12 13 11 11 10 13 12
165362024-02-08T06:45:48.410Z05/30 Flush
165372024-02-08T06:45:48.410Z06/30 Write block 211 len 24576 data: 18 20 18 16 14 14
165382024-02-08T06:45:48.410Z07/30 Write block 170 len 40960 data: 15 12 13 14 11 12 11 10 8 8
165392024-02-08T06:45:48.410Z08/30 Write block 15 len 36864 data: 8 7 7 8 9 12 10 11 11
165402024-02-08T06:45:48.410Z09/30 Read block 158 len 28672
165412024-02-08T06:45:48.410Z10/30 Write block 588 len 12288 data: 9 12 12
165422024-02-08T06:45:48.411Z11/30 Read block 239 len 36864
165432024-02-08T06:45:48.411Z12/30 Flush
165442024-02-08T06:45:48.414Z13/30 Write block 26 len 20480 data: 16 16 15 13 13
165452024-02-08T06:45:48.414Z14/30 Read block 428 len 32768
165462024-02-08T06:45:48.414Z15/30 Write block 348 len 12288 data: 9 10 11
165472024-02-08T06:45:48.414Z16/30 Write block 562 len 28672 data: 15 15 13 13 12 13 14
165482024-02-08T06:45:48.414Z17/30 Read block 238 len 12288
165492024-02-08T06:45:48.414Z18/30 Read block 346 len 36864
165502024-02-08T06:45:48.414Z19/30 Write block 137 len 32768 data: 12 12 11 9 8 7 8 7
165512024-02-08T06:45:48.415Z20/30 Read block 155 len 16384
165522024-02-08T06:45:48.418Z21/30 Read block 523 len 40960
165532024-02-08T06:45:48.418Z22/30 Read block 436 len 24576
165542024-02-08T06:45:48.418Z23/30 Read block 103 len 32768
165552024-02-08T06:45:48.421Z24/30 Write block 216 len 8192 data: 15 15
165562024-02-08T06:45:48.421Z25/30 Write block 401 len 32768 data: 14 13 12 13 14 13 12 12
165572024-02-08T06:45:48.421Z26/30 Read block 99 len 12288
165582024-02-08T06:45:48.422Z27/30 Write block 556 len 8192 data: 19 12
165592024-02-08T06:45:48.422Z28/30 Read block 61 len 4096
165602024-02-08T06:45:48.425Z29/30 Write block 488 len 12288 data: 14 12 13
165612024-02-08T06:45:48.425Z30/30 Write block 177 len 20480 data: 11 9 9 9 14
165622024-02-08T06:45:48.425Z----------------------------------------------------------------
165632024-02-08T06:45:48.425Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:25
165642024-02-08T06:45:48.425ZStates: Active Active Active
165652024-02-08T06:45:48.426ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
165662024-02-08T06:45:48.426Z 6 Acked 1005 Write 6 Done Done Done false
165672024-02-08T06:45:48.426Z 7 Acked 1006 Write 10 Done Done Done false
165682024-02-08T06:45:48.426Z 8 Acked 1007 Write 9 Done Done Done false
165692024-02-08T06:45:48.426Z 9 Acked 1008 Read 7 Done Done Sent false
165702024-02-08T06:45:48.426Z 10 Acked 1009 Write 3 Done Done Sent false
165712024-02-08T06:45:48.426Z 11 Acked 1010 Read 9 Done Done Sent false
165722024-02-08T06:45:48.426Z 12 Acked 1011 Flush 0 Done Done Sent false
165732024-02-08T06:45:48.426Z 13 Acked 1012 Write 5 Done Done Sent false
165742024-02-08T06:45:48.426Z 14 Acked 1013 Read 8 Done Done Sent false
165752024-02-08T06:45:48.426Z 15 Acked 1014 Write 3 Done Done Sent false
165762024-02-08T06:45:48.426Z 16 Acked 1015 Write 7 Done Done Sent false
165772024-02-08T06:45:48.426Z 17 Acked 1016 Read 3 Done Done Sent false
165782024-02-08T06:45:48.426Z 18 Acked 1017 Read 9 Done Done Sent false
165792024-02-08T06:45:48.426Z 19 Acked 1018 Write 8 Done Done Sent false
165802024-02-08T06:45:48.426Z 20 Acked 1019 Read 4 Done Done Sent false
165812024-02-08T06:45:48.427Z 21 Acked 1020 Read 10 Done Done Sent false
165822024-02-08T06:45:48.427Z 22 Acked 1021 Read 6 Done Done Sent false
165832024-02-08T06:45:48.427Z 23 Acked 1022 Read 8 Done Done Sent false
165842024-02-08T06:45:48.427Z 24 Acked 1023 Write 2 Done Done Sent false
165852024-02-08T06:45:48.427Z 25 Acked 1024 Write 8 Done Done Sent false
165862024-02-08T06:45:48.427Z 26 Acked 1025 Read 3 Done Done Sent false
165872024-02-08T06:45:48.427Z 27 Acked 1026 Write 2 Done Done Sent false
165882024-02-08T06:45:48.427Z 28 Acked 1027 Read 1 Done Done Sent false
165892024-02-08T06:45:48.427Z 29 Acked 1028 Write 3 Done Done Sent false
165902024-02-08T06:45:48.427Z 30 Acked 1029 Write 5 Sent Sent Sent false
165912024-02-08T06:45:48.427Z STATES DS:0 DS:1 DS:2 TOTAL
165922024-02-08T06:45:48.427Z New 0 0 0 0
165932024-02-08T06:45:48.427Z Sent 1 1 22 24
165942024-02-08T06:45:48.427Z Done 24 24 3 51
165952024-02-08T06:45:48.427Z Skipped 0 0 0 0
165962024-02-08T06:45:48.428Z Error 0 0 0 0
165972024-02-08T06:45:48.428ZLast Flush: 1011 1011 1004
165982024-02-08T06:45:48.428ZDownstairs last five completed: 1004 1003 1002 1001 1000
165992024-02-08T06:45:48.428ZUpstairs last five completed: 30 29 28 27 26
166002024-02-08T06:45:48.428Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:47.641886153Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1746}
166012024-02-08T06:45:48.428ZWrote out file "/tmp/test_repair_verify.data"
166022024-02-08T06:45:48.431Z
166032024-02-08T06:45:48.431ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166042024-02-08T06:45:48.451ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166052024-02-08T06:45:48.451Z 0 000-019 152 152 143 259 259 240 F F T <---
166062024-02-08T06:45:48.451Z 1 020-039 152 152 152 259 259 258 T T T <---
166072024-02-08T06:45:48.451Z 2 040-059 150 150 150 256 256 256 F F F
166082024-02-08T06:45:48.451Z 3 060-079 150 150 150 255 255 255 F F F
166092024-02-08T06:45:48.451Z 4 080-099 150 150 150 255 255 255 F F F
166102024-02-08T06:45:48.451Z 5 100-119 150 150 150 255 255 255 F F F
166112024-02-08T06:45:48.452Z 6 120-139 144 144 144 244 244 244 T T F <---
166122024-02-08T06:45:48.452Z 7 140-159 151 151 151 257 257 257 T T F <---
166132024-02-08T06:45:48.452Z 8 160-179 152 152 151 259 259 257 T T T <---
166142024-02-08T06:45:48.459Z 9 180-199 149 149 149 254 254 254 T T F <---
166152024-02-08T06:45:48.459Z 10 200-219 152 152 152 259 259 258 T T T <---
166162024-02-08T06:45:48.459Z 11 220-239 148 148 148 253 253 253 F F F
166172024-02-08T06:45:48.459Z 12 240-259 148 148 148 253 253 253 F F F
166182024-02-08T06:45:48.459Z 13 260-279 150 150 150 255 255 255 F F F
166192024-02-08T06:45:48.459Z 14 280-299 148 148 148 253 253 253 F F F
166202024-02-08T06:45:48.459Z 15 300-319 133 133 133 224 224 224 F F F
166212024-02-08T06:45:48.460Z 16 320-339 143 143 143 240 240 240 F F F
166222024-02-08T06:45:48.460Z 17 340-359 146 146 146 248 248 248 T T F <---
166232024-02-08T06:45:48.460Z 18 360-379 150 150 150 255 255 255 F F F
166242024-02-08T06:45:48.460Z 19 380-399 147 147 147 250 250 250 F F F
166252024-02-08T06:45:48.460Z 20 400-419 150 150 150 256 256 256 T T F <---
166262024-02-08T06:45:48.460Z 21 420-439 149 149 149 254 254 254 F F F
166272024-02-08T06:45:48.460Z 22 440-459 146 146 146 249 249 249 F F F
166282024-02-08T06:45:48.460Z 23 460-479 150 150 150 256 256 256 F F F
166292024-02-08T06:45:48.460Z 24 480-499 152 152 152 258 258 258 T T F <---
166302024-02-08T06:45:48.460Z 25 500-519 150 150 150 256 256 256 F F F
166312024-02-08T06:45:48.460Z 26 520-539 148 148 148 253 253 253 F F F
166322024-02-08T06:45:48.460Z 27 540-559 148 148 148 253 253 253 T T F <---
166332024-02-08T06:45:48.460Z 28 560-579 149 149 149 254 254 254 T T F <---
166342024-02-08T06:45:48.460Z 29 580-599 152 152 150 259 259 255 F F F <---
166352024-02-08T06:45:48.460ZMax gen: 152, Max flush: 259
166362024-02-08T06:45:48.461ZError: Difference in extent metadata found!
166372024-02-08T06:45:48.461ZA Difference in extent metadata is expected here
166382024-02-08T06:45:48.461ZOn loop 76
166392024-02-08T06:45:48.461Z
166402024-02-08T06:45:48.461ZVerifying data now
166412024-02-08T06:45:54.110ZNo change after verify
166422024-02-08T06:45:54.110ZLoop: 76 Downstairs dump after verify (and repair):
166432024-02-08T06:45:54.128ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166442024-02-08T06:45:54.128Z 0 000-019 152 152 152 259 259 259 F F F
166452024-02-08T06:45:54.128Z 1 020-039 153 153 153 260 260 260 F F F
166462024-02-08T06:45:54.128Z 2 040-059 150 150 150 256 256 256 F F F
166472024-02-08T06:45:54.128Z 3 060-079 150 150 150 255 255 255 F F F
166482024-02-08T06:45:54.128Z 4 080-099 150 150 150 255 255 255 F F F
166492024-02-08T06:45:54.128Z 5 100-119 150 150 150 255 255 255 F F F
166502024-02-08T06:45:54.128Z 6 120-139 153 153 153 260 260 260 F F F
166512024-02-08T06:45:54.135Z 7 140-159 153 153 153 260 260 260 F F F
166522024-02-08T06:45:54.135Z 8 160-179 153 153 153 260 260 260 F F F
166532024-02-08T06:45:54.135Z 9 180-199 153 153 153 260 260 260 F F F
166542024-02-08T06:45:54.135Z 10 200-219 153 153 153 260 260 260 F F F
166552024-02-08T06:45:54.135Z 11 220-239 148 148 148 253 253 253 F F F
166562024-02-08T06:45:54.135Z 12 240-259 148 148 148 253 253 253 F F F
166572024-02-08T06:45:54.135Z 13 260-279 150 150 150 255 255 255 F F F
166582024-02-08T06:45:54.135Z 14 280-299 148 148 148 253 253 253 F F F
166592024-02-08T06:45:54.135Z 15 300-319 133 133 133 224 224 224 F F F
166602024-02-08T06:45:54.135Z 16 320-339 143 143 143 240 240 240 F F F
166612024-02-08T06:45:54.136Z 17 340-359 153 153 153 260 260 260 F F F
166622024-02-08T06:45:54.136Z 18 360-379 150 150 150 255 255 255 F F F
166632024-02-08T06:45:54.136Z 19 380-399 147 147 147 250 250 250 F F F
166642024-02-08T06:45:54.136Z 20 400-419 153 153 153 260 260 260 F F F
166652024-02-08T06:45:54.136Z 21 420-439 149 149 149 254 254 254 F F F
166662024-02-08T06:45:54.136Z 22 440-459 146 146 146 249 249 249 F F F
166672024-02-08T06:45:54.136Z 23 460-479 150 150 150 256 256 256 F F F
166682024-02-08T06:45:54.136Z 24 480-499 153 153 153 260 260 260 F F F
166692024-02-08T06:45:54.136Z 25 500-519 150 150 150 256 256 256 F F F
166702024-02-08T06:45:54.136Z 26 520-539 148 148 148 253 253 253 F F F
166712024-02-08T06:45:54.136Z 27 540-559 153 153 153 260 260 260 F F F
166722024-02-08T06:45:54.136Z 28 560-579 153 153 153 260 260 260 F F F
166732024-02-08T06:45:54.136Z 29 580-599 152 152 152 259 259 259 F F F
166742024-02-08T06:45:54.136ZMax gen: 153, Max flush: 260
166752024-02-08T06:45:54.136Z
166762024-02-08T06:45:54.137ZBegin loop 77
166772024-02-08T06:45:54.137ZDownstairs to restart: 2
166782024-02-08T06:45:54.149Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.364978065Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754}
166792024-02-08T06:45:54.150Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365273113Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754}
166802024-02-08T06:45:54.150Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365292251Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754}
166812024-02-08T06:45:54.150Z{"msg":"Crucible runtime is spawned
166822024-02-08T06:45:54.568ZCrucible 5a923a02-988b-43ce-9ea1-a4516e40207a has session id: a285263b-80ea-4bf9-b5a6-14bb84bd80dd","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365335648Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166832024-02-08T06:45:54.568Z{"msg":"Upstairs opts: Upstairs UUID: 5a923a02-988b-43ce-9ea1-a4516e40207a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365368355Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166842024-02-08T06:45:54.569Z{"msg":"Crucible stats registered with UUID: 5a923a02-988b-43ce-9ea1-a4516e40207a","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365405683Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166852024-02-08T06:45:54.569Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365420662Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754}
166862024-02-08T06:45:54.569Z{"msg":"5a923a02-988b-43ce-9ea1-a4516e40207a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365566991Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166872024-02-08T06:45:54.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365605278Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166882024-02-08T06:45:54.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365667663Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166892024-02-08T06:45:54.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365719649Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166902024-02-08T06:45:54.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365756857Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166912024-02-08T06:45:54.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365789714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166922024-02-08T06:45:54.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365810562Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166932024-02-08T06:45:54.569Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365892216Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"io task","client":"0","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166942024-02-08T06:45:54.569Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.365940373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"io task","client":"1","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166952024-02-08T06:45:54.569Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.36597031Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"io task","client":"2","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166962024-02-08T06:45:54.569Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.366011407Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"io task","client":"2","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166972024-02-08T06:45:54.569Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.366040365Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"io task","client":"0","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166982024-02-08T06:45:54.570Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.366066873Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"io task","client":"1","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
166992024-02-08T06:45:54.570Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.366579275Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167002024-02-08T06:45:54.570Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.366621512Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167012024-02-08T06:45:54.570Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.366648699Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167022024-02-08T06:45:54.570Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.366688236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167032024-02-08T06:45:54.570Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.3671707Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167042024-02-08T06:45:54.570Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.367332848Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167052024-02-08T06:45:54.570Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.367745187Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167062024-02-08T06:45:54.570Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.36783744Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167072024-02-08T06:45:54.570Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.367862778Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167082024-02-08T06:45:54.570Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.367888956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167092024-02-08T06:45:54.570Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.367910405Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167102024-02-08T06:45:54.570Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.367934883Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167112024-02-08T06:45:54.570Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.368258209Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167122024-02-08T06:45:54.570Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.368295266Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167132024-02-08T06:45:54.571Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.36876749Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167142024-02-08T06:45:54.571Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369205687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167152024-02-08T06:45:54.571Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369231965Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167162024-02-08T06:45:54.571Z{"msg":"[0]R flush_numbers[0..12]: [259, 260, 256, 255, 255, 255, 260, 260, 260, 260, 260, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369246584Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167172024-02-08T06:45:54.571Z{"msg":"[0]R generation[0..12]: [152, 153, 150, 150, 150, 150, 153, 153, 153, 153, 153, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369260863Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167182024-02-08T06:45:54.571Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369282782Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167192024-02-08T06:45:54.571Z{"msg":"[1]R flush_numbers[0..12]: [259, 260, 256, 255, 255, 255, 260, 260, 260, 260, 260, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.36929782Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167202024-02-08T06:45:54.571Z{"msg":"[1]R generation[0..12]: [152, 153, 150, 150, 150, 150, 153, 153, 153, 153, 153, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369312639Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167212024-02-08T06:45:54.571Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369333348Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167222024-02-08T06:45:54.571Z{"msg":"[2]R flush_numbers[0..12]: [259, 260, 256, 255, 255, 255, 260, 260, 260, 260, 260, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369347687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167232024-02-08T06:45:54.571Z{"msg":"[2]R generation[0..12]: [152, 153, 150, 150, 150, 150, 153, 153, 153, 153, 153, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369373535Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167242024-02-08T06:45:54.571Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369388264Z"Wait for a query_work_queue command to finish before sending IO
167252024-02-08T06:45:54.571Z,"hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167262024-02-08T06:45:54.571Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369406962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167272024-02-08T06:45:54.571Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369427041Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167282024-02-08T06:45:54.572Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.36944024Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167292024-02-08T06:45:54.572Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369464928Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167302024-02-08T06:45:54.572Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369498705Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167312024-02-08T06:45:54.572Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369515174Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167322024-02-08T06:45:54.572Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369549182Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167332024-02-08T06:45:54.572Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369584969Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167342024-02-08T06:45:54.572Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369619956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167352024-02-08T06:45:54.572Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369644025Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167362024-02-08T06:45:54.572Z{"msg":"5a923a02-988b-43ce-9ea1-a4516e40207a is now active with session: a285263b-80ea-4bf9-b5a6-14bb84bd80dd","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369676692Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167372024-02-08T06:45:54.572Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.36970725Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754,"session_id":"a285263b-80ea-4bf9-b5a6-14bb84bd80dd"}
167382024-02-08T06:45:54.572ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
167392024-02-08T06:45:54.572Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:53.369742747Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754}
167402024-02-08T06:45:54.572ZLoading write count information from file "/tmp/test_repair_verify.data"
167412024-02-08T06:45:54.572ZRun Repair workload
167422024-02-08T06:45:54.572Z01/30 Write block 580 len 8192 data: 10 10
167432024-02-08T06:45:54.572Z02/30 Flush
167442024-02-08T06:45:54.582Z03/30 Write block 300 len 12288 data: 14 16 17
167452024-02-08T06:45:54.583Z04/30 Read block 416 len 40960
167462024-02-08T06:45:54.583Z05/30 Read block 39 len 32768
167472024-02-08T06:45:54.583Z06/30 Write block 590 len 24576 data: 13 13 11 12 11 8
167482024-02-08T06:45:54.583Z07/30 Write block 109 len 36864 data: 11 7 8 8 12 11 12 13 10
167492024-02-08T06:45:54.584Z08/30 Read block 570 len 24576
167502024-02-08T06:45:54.584Z09/30 Write block 460 len 16384 data: 10 11 14 14
167512024-02-08T06:45:54.584Z10/30 Read block 473 len 24576
167522024-02-08T06:45:54.584Z11/30 Flush
167532024-02-08T06:45:54.584Z12/30 Write block 224 len 32768 data: 17 16 16 16 17 15 12 9
167542024-02-08T06:45:54.584Z13/30 Read block 443 len 20480
167552024-02-08T06:45:54.585Z14/30 Write block 378 len 20480 data: 9 9 9 6 9
167562024-02-08T06:45:54.585Z15/30 Flush
167572024-02-08T06:45:54.585Z16/30 Write block 71 len 16384 data: 9 9 9 13
167582024-02-08T06:45:54.585Z17/30 Read block 586 len 20480
167592024-02-08T06:45:54.585Z18/30 Read block 550 len 20480
167602024-02-08T06:45:54.585Z19/30 Write block 157 len 40960 data: 13 14 14 12 13 15 15 15 17 18
167612024-02-08T06:45:54.586Z20/30 Write block 112 len 32768 data: 9 13 12 13 14 11 9 9
167622024-02-08T06:45:54.586Z21/30 Read block 517 len 4096
167632024-02-08T06:45:54.586Z22/30 Write block 360 len 40960 data: 10 7 8 8 13 14 13 15 12 14
167642024-02-08T06:45:54.586Z23/30 Read block 53 len 12288
167652024-02-08T06:45:54.586Z24/30 Write block 467 len 8192 data: 11 11
167662024-02-08T06:45:54.586Z25/30 Write block 121 len 28672 data: 8 7 8 8 8 6 5
167672024-02-08T06:45:54.587Z26/30 Write block 352 len 32768 data: 10 9 10 10 9 10 9 11
167682024-02-08T06:45:54.587Z27/30 Read block 11 len 16384
167692024-02-08T06:45:54.587Z28/30 Read block 216 len 28672
167702024-02-08T06:45:54.587Z29/30 Write block 440 len 24576 data: 12 12 12 13 12 13
167712024-02-08T06:45:54.587Z30/30 Read block 168 len 16384
167722024-02-08T06:45:54.587Z----------------------------------------------------------------
167732024-02-08T06:45:54.588Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:28
167742024-02-08T06:45:54.588ZStates: Active Active Active
167752024-02-08T06:45:54.588ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
167762024-02-08T06:45:54.588Z 3 Acked 1002 Write 3 Done Done Sent false
167772024-02-08T06:45:54.588Z 4 Acked 1003 Read 10 Done Done Sent false
167782024-02-08T06:45:54.588Z 5 Acked 1004 Read 8 Done Done Sent false
167792024-02-08T06:45:54.588Z 6 Acked 1005 Write 6 Done Done Sent false
167802024-02-08T06:45:54.588Z 7 Acked 1006 Write 9 Done Done Sent false
167812024-02-08T06:45:54.588Z 8 Acked 1007 Read 6 Done Done Sent false
167822024-02-08T06:45:54.588Z 9 Acked 1008 Write 4 Done Done Sent false
167832024-02-08T06:45:54.588Z 10 Acked 1009 Read 6 Done Done Sent false
167842024-02-08T06:45:54.589Z 11 Acked 1010 Flush 0 Done Done Sent false
167852024-02-08T06:45:54.589Z 12 Acked 1011 Write 8 Done Done Sent false
167862024-02-08T06:45:54.589Z 13 Acked 1012 Read 5 Done Done Sent false
167872024-02-08T06:45:54.589Z 14 Acked 1013 Write 5 Done Done Sent false
167882024-02-08T06:45:54.589Z 15 Acked 1014 Flush 0 Done Done Sent false
167892024-02-08T06:45:54.589Z 16 Acked 1015 Write 4 Done Done Sent false
167902024-02-08T06:45:54.589Z 17 Acked 1016 Read 5 Done Done Sent false
167912024-02-08T06:45:54.589Z 18 Acked 1017 Read 5 Done Done Sent false
167922024-02-08T06:45:54.589Z 19 Acked 1018 Write 10 Done Done Sent false
167932024-02-08T06:45:54.589Z 20 Acked 1019 Write 8 Done Done Sent false
167942024-02-08T06:45:54.590Z 21 Acked 1020 Read 1 Done Done Sent false
167952024-02-08T06:45:54.590Z 22 Acked 1021 Write 10 Done Done Sent false
167962024-02-08T06:45:54.590Z 23 Acked 1022 Read 3 Done Done Sent false
167972024-02-08T06:45:54.590Z 24 Acked 1023 Write 2 Done Done Sent false
167982024-02-08T06:45:54.590Z 25 Acked 1024 Write 7 Done Done Sent false
167992024-02-08T06:45:54.590Z 26 Acked 1025 Write 8 Done Done Sent false
168002024-02-08T06:45:54.590Z 27 Acked 1026 Read 4 Done Done Sent false
168012024-02-08T06:45:54.590Z 28 Acked 1027 Read 7 Done Done Sent false
168022024-02-08T06:45:54.590Z 29 Acked 1028 Write 6 Done Done Sent false
168032024-02-08T06:45:54.590Z 30 Acked 1029 Read 4 Done Done Sent false
168042024-02-08T06:45:54.590Z STATES DS:0 DS:1 DS:2 TOTAL
168052024-02-08T06:45:54.590Z New 0 0 0 0
168062024-02-08T06:45:54.591Z Sent 0 0 28 28
168072024-02-08T06:45:54.591Z Done 28 28 0 56
168082024-02-08T06:45:54.591Z Skipped 0 0 0 0
168092024-02-08T06:45:54.591Z Error 0 0 0 0
168102024-02-08T06:45:54.591ZLast Flush: 1014 1014 1001
168112024-02-08T06:45:54.591ZDownstairs last five completed: 1001 1000
168122024-02-08T06:45:54.591ZUpstairs last five completed: 30 29 28 27 26
168132024-02-08T06:45:54.591Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:53.402543324Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1754}
168142024-02-08T06:45:54.591ZWrote out file "/tmp/test_repair_verify.data"
168152024-02-08T06:45:54.591Z
168162024-02-08T06:45:54.592ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168172024-02-08T06:45:54.592ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168182024-02-08T06:45:54.592Z 0 000-019 152 152 152 259 259 259 F F F
168192024-02-08T06:45:54.592Z 1 020-039 153 153 153 260 260 260 F F F
168202024-02-08T06:45:54.592Z 2 040-059 150 150 150 256 256 256 F F F
168212024-02-08T06:45:54.592Z 3 060-079 150 150 150 255 255 255 T T F <---
168222024-02-08T06:45:54.592Z 4 080-099 150 150 150 255 255 255 F F F
168232024-02-08T06:45:54.592Z 5 100-119 154 154 150 262 262 255 T T F <---
168242024-02-08T06:45:54.592Z 6 120-139 153 153 153 260 260 260 T T F <---
168252024-02-08T06:45:54.592Z 7 140-159 153 153 153 260 260 260 T T F <---
168262024-02-08T06:45:54.592Z 8 160-179 153 153 153 260 260 260 T T F <---
168272024-02-08T06:45:54.592Z 9 180-199 153 153 153 260 260 260 F F F
168282024-02-08T06:45:54.593Z 10 200-219 153 153 153 260 260 260 F F F
168292024-02-08T06:45:54.593Z 11 220-239 154 154 148 263 263 253 F F F <---
168302024-02-08T06:45:54.593Z 12 240-259 148 148 148 253 253 253 F F F
168312024-02-08T06:45:54.593Z 13 260-279 150 150 150 255 255 255 F F F
168322024-02-08T06:45:54.593Z 14 280-299 148 148 148 253 253 253 F F F
168332024-02-08T06:45:54.593Z 15 300-319 154 154 133 262 262 224 F F F <---
168342024-02-08T06:45:54.593Z 16 320-339 143 143 143 240 240 240 F F F
168352024-02-08T06:45:54.593Z 17 340-359 153 153 153 260 260 260 T T F <---
168362024-02-08T06:45:54.593Z 18 360-379 154 154 150 263 263 255 T T F <---
168372024-02-08T06:45:54.593Z 19 380-399 154 154 147 263 263 250 F F F <---
168382024-02-08T06:45:54.593Z 20 400-419 153 153 153 260 260 260 F F F
168392024-02-08T06:45:54.593Z 21 420-439 149 149 149 254 254 254 F F F
168402024-02-08T06:45:54.593Z 22 440-459 146 146 146 249 249 249 T T F <---
168412024-02-08T06:45:54.594Z 23 460-479 154 154 150 262 262 256 T T F <---
168422024-02-08T06:45:54.594Z 24 480-499 153 153 153 260 260 260 F F F
168432024-02-08T06:45:54.594Z 25 500-519 150 150 150 256 256 256 F F F
168442024-02-08T06:45:55.013Z 26 520-539 148 148 148 253 253 253 F F F
168452024-02-08T06:45:55.013Z 27 540-559 153 153 153 260 260 260 F F F
168462024-02-08T06:45:55.014Z 28 560-579 153 153 153 260 260 260 F F F
168472024-02-08T06:45:55.014Z 29 580-599 154 154 154 262 262 261 F F F <---
168482024-02-08T06:45:55.014ZMax gen: 154, Max flush: 263
168492024-02-08T06:45:55.014ZError: Difference in extent metadata found!
168502024-02-08T06:45:55.015ZA Difference in extent metadata is expected here
168512024-02-08T06:45:55.015ZOn loop 77
168522024-02-08T06:45:55.015Z
168532024-02-08T06:45:55.015ZVerifying data now
168542024-02-08T06:45:58.859ZNo change after verify
168552024-02-08T06:45:58.860ZLoop: 77 Downstairs dump after verify (and repair):
168562024-02-08T06:45:58.877ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168572024-02-08T06:45:58.877Z 0 000-019 152 152 152 259 259 259 F F F
168582024-02-08T06:45:58.877Z 1 020-039 153 153 153 260 260 260 F F F
168592024-02-08T06:45:58.877Z 2 040-059 150 150 150 256 256 256 F F F
168602024-02-08T06:45:58.877Z 3 060-079 155 155 155 264 264 264 F F F
168612024-02-08T06:45:58.878Z 4 080-099 150 150 150 255 255 255 F F F
168622024-02-08T06:45:58.878Z 5 100-119 155 155 155 264 264 264 F F F
168632024-02-08T06:45:58.878Z 6 120-139 155 155 155 264 264 264 F F F
168642024-02-08T06:45:58.878Z 7 140-159 155 155 155 264 264 264 F F F
168652024-02-08T06:45:58.878Z 8 160-179 155 155 155 264 264 264 F F F
168662024-02-08T06:45:58.878Z 9 180-199 153 153 153 260 260 260 F F F
168672024-02-08T06:45:58.881Z 10 200-219 153 153 153 260 260 260 F F F
168682024-02-08T06:45:58.881Z 11 220-239 154 154 154 263 263 263 F F F
168692024-02-08T06:45:58.881Z 12 240-259 148 148 148 253 253 253 F F F
168702024-02-08T06:45:58.882Z 13 260-279 150 150 150 255 255 255 F F F
168712024-02-08T06:45:58.882Z 14 280-299 148 148 148 253 253 253 F F F
168722024-02-08T06:45:58.882Z 15 300-319 154 154 154 262 262 262 F F F
168732024-02-08T06:45:58.882Z 16 320-339 143 143 143 240 240 240 F F F
168742024-02-08T06:45:58.882Z 17 340-359 155 155 155 264 264 264 F F F
168752024-02-08T06:45:58.882Z 18 360-379 155 155 155 264 264 264 F F F
168762024-02-08T06:45:58.882Z 19 380-399 154 154 154 263 263 263 F F F
168772024-02-08T06:45:58.882Z 20 400-419 153 153 153 260 260 260 F F F
168782024-02-08T06:45:58.882Z 21 420-439 149 149 149 254 254 254 F F F
168792024-02-08T06:45:58.882Z 22 440-459 155 155 155 264 264 264 F F F
168802024-02-08T06:45:58.882Z 23 460-479 155 155 155 264 264 264 F F F
168812024-02-08T06:45:58.882Z 24 480-499 153 153 153 260 260 260 F F F
168822024-02-08T06:45:58.882Z 25 500-519 150 150 150 256 256 256 F F F
168832024-02-08T06:45:58.882Z 26 520-539 148 148 148 253 253 253 F F F
168842024-02-08T06:45:58.882Z 27 540-559 153 153 153 260 260 260 F F F
168852024-02-08T06:45:58.883Z 28 560-579 153 153 153 260 260 260 F F F
168862024-02-08T06:45:58.883Z 29 580-599 154 154 154 262 262 262 F F F
168872024-02-08T06:45:58.883ZMax gen: 155, Max flush: 264
168882024-02-08T06:45:58.883Z
168892024-02-08T06:45:58.883ZBegin loop 78
168902024-02-08T06:45:58.883ZDownstairs to restart: 1
168912024-02-08T06:45:58.895Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.112945088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762}
168922024-02-08T06:45:58.895Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113197549Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762}
168932024-02-08T06:45:58.895Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113226267Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762}
168942024-02-08T06:45:58.901Z{"msg":"Crucible ccd21379-81c7-437b-b985-ea30366f0458 has session id: 06756f57-a4ec-4183-9143-7414320c0859"Crucible runtime is spawned
168952024-02-08T06:45:58.901Z,"v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113266724Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
168962024-02-08T06:45:58.901Z{"msg":"Upstairs opts: Upstairs UUID: ccd21379-81c7-437b-b985-ea30366f0458, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113298551Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
168972024-02-08T06:45:58.901Z{"msg":"Crucible stats registered with UUID: ccd21379-81c7-437b-b985-ea30366f0458","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113333329Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
168982024-02-08T06:45:58.901Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113348058Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762}
168992024-02-08T06:45:58.901Z{"msg":"ccd21379-81c7-437b-b985-ea30366f0458 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113499866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169002024-02-08T06:45:58.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113537684Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169012024-02-08T06:45:58.901Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113569981Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169022024-02-08T06:45:58.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113614098Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169032024-02-08T06:45:58.901Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113650695Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169042024-02-08T06:45:58.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113671734Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169052024-02-08T06:45:58.901Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113705271Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169062024-02-08T06:45:58.901Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113730309Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"io task","client":"2","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169072024-02-08T06:45:58.902Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113824692Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"io task","client":"1","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169082024-02-08T06:45:58.902Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113863169Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169092024-02-08T06:45:58.902Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113888477Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"io task","client":"2","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169102024-02-08T06:45:58.902Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113924235Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169112024-02-08T06:45:58.902Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.113949962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"io task","client":"1","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169122024-02-08T06:45:58.902Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.114538309Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169132024-02-08T06:45:58.902Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.114567236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169142024-02-08T06:45:58.902Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.114583525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169152024-02-08T06:45:58.902Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.114613993Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169162024-02-08T06:45:58.902Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.115214888Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169172024-02-08T06:45:58.902Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.115360907Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169182024-02-08T06:45:58.902Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.115592699Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169192024-02-08T06:45:58.902Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.115623607Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169202024-02-08T06:45:58.902Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.115802314Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169212024-02-08T06:45:58.902Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.11584168Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169222024-02-08T06:45:58.903Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.115864559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169232024-02-08T06:45:58.903Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.115909586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169242024-02-08T06:45:58.903Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.115935764Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169252024-02-08T06:45:58.903Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.115961662Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169262024-02-08T06:45:58.903Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116158857Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169272024-02-08T06:45:58.903Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116609593Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169282024-02-08T06:45:58.903Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.11664462Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169292024-02-08T06:45:58.903Z{"msg":"[0]R flush_numbers[0..12]: [259, 260, 256, 264, 255, 264, 264, 264, 264, 260, 260, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116661709Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169302024-02-08T06:45:58.903Z{"msg":"[0]R generation[0..12]: [152, 153, 150, 155, 150, 155, 155, 155, 155, 153, 153, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116676438Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169312024-02-08T06:45:58.903Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116690987Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169322024-02-08T06:45:58.908Z{"msg":"[1]R flush_numbers[0..12]: [259, 260, 256, 264, 255, 264, 264, 264, 264, 260, 260, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116733194Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169332024-02-08T06:45:58.908Z{"msg":"[1]R generation[0..12]: [152, 153, 150, 155, 150, 155, 155, 155, 155, 153, 153, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116758112Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169342024-02-08T06:45:58.908Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116773381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169352024-02-08T06:45:58.908Z{"msg":"[2]R flush_numbers[0..12]: [259, 260, 256, 264, 255, 264, 264, 264, 264, 260, 260, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116787469Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169362024-02-08T06:45:58.908Z{"msg":"[2]R generation[0..12]: [152, 153, 150, 155, 150, 155, 155, 155, 155, 153, 153, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116808688Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169372024-02-08T06:45:58.908Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116823137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169382024-02-08T06:45:58.908Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116836346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169392024-02-08T06:45:58.908Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116856784Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169402024-02-08T06:45:58.909Z{"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116870213Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169412024-02-08T06:45:58.909Z{"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
169422024-02-08T06:45:58.909Z,"time":"2024-02-08T06:45:58.116883982Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169432024-02-08T06:45:58.909Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116900141Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169442024-02-08T06:45:58.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.11691237Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169452024-02-08T06:45:58.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116926629Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169462024-02-08T06:45:58.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116977905Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169472024-02-08T06:45:58.909Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.116991674Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169482024-02-08T06:45:58.909Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.117004023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169492024-02-08T06:45:58.909Z{"msg":"ccd21379-81c7-437b-b985-ea30366f0458 is now active with session: 06756f57-a4ec-4183-9143-7414320c0859","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.117024912Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169502024-02-08T06:45:58.909Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.117037221Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762,"session_id":"06756f57-a4ec-4183-9143-7414320c0859"}
169512024-02-08T06:45:58.909Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:45:58.11704972Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762}
169522024-02-08T06:45:58.909ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169532024-02-08T06:45:58.909ZLoading write count information from file "/tmp/test_repair_verify.data"
169542024-02-08T06:45:58.909ZRun Repair workload
169552024-02-08T06:45:58.910Z01/30 Write block 548 len 32768 data: 19 21 21 20 18 17 16 16
169562024-02-08T06:45:58.910Z02/30 Write block 425 len 20480 data: 9 9 6 6 6
169572024-02-08T06:45:58.910Z03/30 Write block 444 len 4096 data: 13
169582024-02-08T06:45:58.910Z04/30 Read block 435 len 28672
169592024-02-08T06:45:58.910Z05/30 Read block 280 len 8192
169602024-02-08T06:45:58.910Z06/30 Read block 259 len 40960
169612024-02-08T06:45:58.910Z07/30 Read block 470 len 36864
169622024-02-08T06:45:58.910Z08/30 Read block 322 len 40960
169632024-02-08T06:45:58.913Z09/30 Read block 503 len 12288
169642024-02-08T06:45:58.913Z10/30 Read block 10 len 32768
169652024-02-08T06:45:58.913Z11/30 Flush
169662024-02-08T06:45:58.913Z12/30 Write block 524 len 16384 data: 11 11 11 10
169672024-02-08T06:45:58.913Z13/30 Read block 63 len 8192
169682024-02-08T06:45:58.913Z14/30 Read block 218 len 32768
169692024-02-08T06:45:58.916Z15/30 Flush
169702024-02-08T06:45:58.916Z16/30 Write block 118 len 24576 data: 10 10 9 9 8 9
169712024-02-08T06:45:58.916Z17/30 Read block 165 len 8192
169722024-02-08T06:45:58.919Z18/30 Read block 363 len 32768
169732024-02-08T06:45:58.919Z19/30 Write block 12 len 32768 data: 9 9 8 9 8 8 9 10
169742024-02-08T06:45:58.919Z20/30 Read block 79 len 8192
169752024-02-08T06:45:58.922Z21/30 Read block 194 len 36864
169762024-02-08T06:45:58.922Z22/30 Write block 82 len 32768 data: 13 12 12 12 12 11 11 10
169772024-02-08T06:45:58.922Z23/30 Flush
169782024-02-08T06:45:58.925Z24/30 Write block 64 len 40960 data: 8 8 7 8 13 14 9 10 10 10
169792024-02-08T06:45:58.928Z25/30 Write block 548 len 24576 data: 20 22 22 21 19 18
169802024-02-08T06:45:58.928Z26/30 Read block 472 len 20480
169812024-02-08T06:45:58.929Z27/30 Write block 345 len 36864 data: 13 14 11 10 11 12 11 11 10
169822024-02-08T06:45:58.929Z28/30 Write block 125 len 24576 data: 9 7 6 8 9 10
169832024-02-08T06:45:58.932Z29/30 Write block 72 len 8192 data: 11 11
169842024-02-08T06:45:58.932Z30/30 Write block 510 len 36864 data: 11 8 8 10 8 7 8 9 10
169852024-02-08T06:45:58.932Z----------------------------------------------------------------
169862024-02-08T06:45:58.932Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
169872024-02-08T06:45:58.932ZStates: Active Active Active
169882024-02-08T06:45:58.932ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
169892024-02-08T06:45:58.932Z 1 Acked 1000 Write 8 Done Sent Done false
169902024-02-08T06:45:58.933Z 2 Acked 1001 Write 5 Done Sent Done false
169912024-02-08T06:45:58.933Z 3 Acked 1002 Write 1 Done Sent Done false
169922024-02-08T06:45:58.933Z 4 Acked 1003 Read 7 Done Sent Done false
169932024-02-08T06:45:58.933Z 5 Acked 1004 Read 2 Done Sent Done false
169942024-02-08T06:45:58.933Z 6 Acked 1005 Read 10 Done Sent Done false
169952024-02-08T06:45:58.933Z 7 Acked 1006 Read 9 Done Sent Done false
169962024-02-08T06:45:58.933Z 8 Acked 1007 Read 10 Done Sent Done false
169972024-02-08T06:45:58.933Z 9 Acked 1008 Read 3 Done Sent Done false
169982024-02-08T06:45:58.933Z 10 Acked 1009 Read 8 Done Sent Done false
169992024-02-08T06:45:58.933Z 11 Acked 1010 Flush 0 Done Sent Done false
170002024-02-08T06:45:58.933Z 12 Acked 1011 Write 4 Done Sent Done false
170012024-02-08T06:45:58.933Z 13 Acked 1012 Read 2 Done Sent Done false
170022024-02-08T06:45:58.933Z 14 Acked 1013 Read 8 Done Sent Done false
170032024-02-08T06:45:58.933Z 15 Acked 1014 Flush 0 Done Sent Done false
170042024-02-08T06:45:58.933Z 16 Acked 1015 Write 6 Done Sent Done false
170052024-02-08T06:45:58.934Z 17 Acked 1016 Read 2 Done Sent Done false
170062024-02-08T06:45:58.934Z 18 Acked 1017 Read 8 Done Sent Done false
170072024-02-08T06:45:58.934Z 19 Acked 1018 Write 8 Done Sent Done false
170082024-02-08T06:45:58.934Z 20 Acked 1019 Read 2 Done Sent Done false
170092024-02-08T06:45:58.934Z 21 Acked 1020 Read 9 Done Sent Done false
170102024-02-08T06:45:58.934Z 22 Acked 1021 Write 8 Done Sent Done false
170112024-02-08T06:45:58.934Z 23 Acked 1022 Flush 0 Done Sent Done false
170122024-02-08T06:45:58.934Z 24 Acked 1023 Write 10 Done Sent Done false
170132024-02-08T06:45:58.934Z 25 Acked 1024 Write 6 Done Sent Done false
170142024-02-08T06:45:58.934Z 26 Acked 1025 Read 5 Done Sent Done false
170152024-02-08T06:45:58.934Z 27 Acked 1026 Write 9 Done Sent Done false
170162024-02-08T06:45:58.934Z 28 Acked 1027 Write 6 Done Sent Done false
170172024-02-08T06:45:58.934Z 29 Acked 1028 Write 2 Done Sent Sent false
170182024-02-08T06:45:58.934Z 30 Acked 1029 Write 9 Sent Sent Sent false
170192024-02-08T06:45:58.934Z STATES DS:0 DS:1 DS:2 TOTAL
170202024-02-08T06:45:58.935Z New 0 0 0 0
170212024-02-08T06:45:58.935Z Sent 1 30 2 33
170222024-02-08T06:45:58.935Z Done 29 0 28 57
170232024-02-08T06:45:58.935Z Skipped 0 0 0 0
170242024-02-08T06:45:58.935Z Error 0 0 0 0
170252024-02-08T06:45:58.935ZLast Flush: 1022 0 1022
170262024-02-08T06:45:58.935ZDownstairs last five completed:
170272024-02-08T06:45:58.935ZUpstairs last five completed: 30 29 28 27 26
170282024-02-08T06:45:58.935Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:45:58.148236059Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1762}
170292024-02-08T06:45:58.935ZWrote out file "/tmp/test_repair_verify.data"
170302024-02-08T06:45:58.939Z
170312024-02-08T06:45:58.939ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
170322024-02-08T06:45:58.954ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170332024-02-08T06:45:58.954Z 0 000-019 156 152 156 267 259 267 F F F <---
170342024-02-08T06:45:58.957Z 1 020-039 153 153 153 260 260 260 F F F
170352024-02-08T06:45:58.957Z 2 040-059 150 150 150 256 256 256 F F F
170362024-02-08T06:45:58.957Z 3 060-079 155 155 155 264 264 264 T F T <---
170372024-02-08T06:45:58.957Z 4 080-099 156 150 156 267 255 267 F F F <---
170382024-02-08T06:45:58.957Z 5 100-119 156 155 156 267 264 267 F F F <---
170392024-02-08T06:45:58.957Z 6 120-139 156 155 156 267 264 267 T F T <---
170402024-02-08T06:45:58.957Z 7 140-159 155 155 155 264 264 264 F F F
170412024-02-08T06:45:58.958Z 8 160-179 155 155 155 264 264 264 F F F
170422024-02-08T06:45:58.958Z 9 180-199 153 153 153 260 260 260 F F F
170432024-02-08T06:45:58.958Z 10 200-219 153 153 153 260 260 260 F F F
170442024-02-08T06:45:58.958Z 11 220-239 154 154 154 263 263 263 F F F
170452024-02-08T06:45:58.958Z 12 240-259 148 148 148 253 253 253 F F F
170462024-02-08T06:45:58.958Z 13 260-279 150 150 150 255 255 255 F F F
170472024-02-08T06:45:58.958Z 14 280-299 148 148 148 253 253 253 F F F
170482024-02-08T06:45:58.958Z 15 300-319 154 154 154 262 262 262 F F F
170492024-02-08T06:45:58.958Z 16 320-339 143 143 143 240 240 240 F F F
170502024-02-08T06:45:58.958Z 17 340-359 155 155 155 264 264 264 T F T <---
170512024-02-08T06:45:58.958Z 18 360-379 155 155 155 264 264 264 F F F
170522024-02-08T06:45:58.958Z 19 380-399 154 154 154 263 263 263 F F F
170532024-02-08T06:45:58.958Z 20 400-419 153 153 153 260 260 260 F F F
170542024-02-08T06:45:58.958Z 21 420-439 156 149 156 265 254 265 F F F <---
170552024-02-08T06:45:58.958Z 22 440-459 156 155 156 265 264 265 F F F <---
170562024-02-08T06:45:58.959Z 23 460-479 155 155 155 264 264 264 F F F
170572024-02-08T06:45:58.959Z 24 480-499 153 153 153 260 260 260 F F F
170582024-02-08T06:45:58.959Z 25 500-519 150 150 150 256 256 256 T F T <---
170592024-02-08T06:45:58.959Z 26 520-539 156 148 156 266 253 266 F F F <---
170602024-02-08T06:45:58.959Z 27 540-559 156 153 156 265 260 265 T F T <---
170612024-02-08T06:45:58.959Z 28 560-579 153 153 153 260 260 260 F F F
170622024-02-08T06:45:58.959Z 29 580-599 154 154 154 262 262 262 F F F
170632024-02-08T06:45:58.959ZMax gen: 156, Max flush: 267
170642024-02-08T06:45:58.959ZError: Difference in extent metadata found!
170652024-02-08T06:45:58.959ZA Difference in extent metadata is expected here
170662024-02-08T06:45:58.959ZOn loop 78
170672024-02-08T06:45:58.959Z
170682024-02-08T06:45:58.965ZVerifying data now
170692024-02-08T06:46:03.535ZNo change after verify
170702024-02-08T06:46:03.535ZLoop: 78 Downstairs dump after verify (and repair):
170712024-02-08T06:46:03.546ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170722024-02-08T06:46:03.547Z 0 000-019 156 156 156 267 267 267 F F F
170732024-02-08T06:46:03.547Z 1 020-039 153 153 153 260 260 260 F F F
170742024-02-08T06:46:03.547Z 2 040-059 150 150 150 256 256 256 F F F
170752024-02-08T06:46:03.547Z 3 060-079 157 157 157 268 268 268 F F F
170762024-02-08T06:46:03.550Z 4 080-099 156 156 156 267 267 267 F F F
170772024-02-08T06:46:03.550Z 5 100-119 156 156 156 267 267 267 F F F
170782024-02-08T06:46:03.550Z 6 120-139 157 157 157 268 268 268 F F F
170792024-02-08T06:46:03.551Z 7 140-159 155 155 155 264 264 264 F F F
170802024-02-08T06:46:03.551Z 8 160-179 155 155 155 264 264 264 F F F
170812024-02-08T06:46:03.551Z 9 180-199 153 153 153 260 260 260 F F F
170822024-02-08T06:46:03.551Z 10 200-219 153 153 153 260 260 260 F F F
170832024-02-08T06:46:03.551Z 11 220-239 154 154 154 263 263 263 F F F
170842024-02-08T06:46:03.551Z 12 240-259 148 148 148 253 253 253 F F F
170852024-02-08T06:46:03.551Z 13 260-279 150 150 150 255 255 255 F F F
170862024-02-08T06:46:03.551Z 14 280-299 148 148 148 253 253 253 F F F
170872024-02-08T06:46:03.551Z 15 300-319 154 154 154 262 262 262 F F F
170882024-02-08T06:46:03.551Z 16 320-339 143 143 143 240 240 240 F F F
170892024-02-08T06:46:03.551Z 17 340-359 157 157 157 268 268 268 F F F
170902024-02-08T06:46:03.551Z 18 360-379 155 155 155 264 264 264 F F F
170912024-02-08T06:46:03.551Z 19 380-399 154 154 154 263 263 263 F F F
170922024-02-08T06:46:03.551Z 20 400-419 153 153 153 260 260 260 F F F
170932024-02-08T06:46:03.551Z 21 420-439 156 156 156 265 265 265 F F F
170942024-02-08T06:46:03.552Z 22 440-459 156 156 156 265 265 265 F F F
170952024-02-08T06:46:03.552Z 23 460-479 155 155 155 264 264 264 F F F
170962024-02-08T06:46:03.552Z 24 480-499 153 153 153 260 260 260 F F F
170972024-02-08T06:46:03.552Z 25 500-519 157 157 157 268 268 268 F F F
170982024-02-08T06:46:03.552Z 26 520-539 156 156 156 266 266 266 F F F
170992024-02-08T06:46:03.552Z 27 540-559 157 157 157 268 268 268 F F F
171002024-02-08T06:46:03.552Z 28 560-579 153 153 153 260 260 260 F F F
171012024-02-08T06:46:03.552Z 29 580-599 154 154 154 262 262 262 F F F
171022024-02-08T06:46:03.552ZMax gen: 157, Max flush: 268
171032024-02-08T06:46:03.552Z
171042024-02-08T06:46:03.552ZBegin loop 79
171052024-02-08T06:46:03.552ZDownstairs to restart: 0
171062024-02-08T06:46:03.567Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.783994155Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770}
171072024-02-08T06:46:03.567Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0Crucible runtime is spawned
171082024-02-08T06:46:03.572Z\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.78432032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770}
171092024-02-08T06:46:03.573Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784354578Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770}
171102024-02-08T06:46:03.573Z{"msg":"Crucible 5d19a2d4-59f5-4b74-a8bb-39ff5438ab34 has session id: efd194b9-e382-4771-bd7d-5031f4fac3ce","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784390925Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171112024-02-08T06:46:03.573Z{"msg":"Upstairs opts: Upstairs UUID: 5d19a2d4-59f5-4b74-a8bb-39ff5438ab34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784418543Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171122024-02-08T06:46:03.573Z{"msg":"Crucible stats registered with UUID: 5d19a2d4-59f5-4b74-a8bb-39ff5438ab34","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784438781Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171132024-02-08T06:46:03.573Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.78445569Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770}
171142024-02-08T06:46:03.573Z{"msg":"5d19a2d4-59f5-4b74-a8bb-39ff5438ab34 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784546623Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171152024-02-08T06:46:03.573Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784577591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171162024-02-08T06:46:03.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784621437Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171172024-02-08T06:46:03.573Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784648866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171182024-02-08T06:46:03.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784675733Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171192024-02-08T06:46:03.573Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784698972Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171202024-02-08T06:46:03.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784734869Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171212024-02-08T06:46:03.573Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784768837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171222024-02-08T06:46:03.573Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784803384Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"io task","client":"1","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171232024-02-08T06:46:03.573Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784828742Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"io task","client":"2","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171242024-02-08T06:46:03.574Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:02.784867379Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171252024-02-08T06:46:03.574Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:02.784906266Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171262024-02-08T06:46:03.574Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784943803Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"io task","client":"2","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171272024-02-08T06:46:03.574Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.784969902Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"io task","client":"1","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171282024-02-08T06:46:03.574Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:02.785017028Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171292024-02-08T06:46:03.574Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.785041136Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171302024-02-08T06:46:03.574Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.785066034Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171312024-02-08T06:46:03.574Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.785091302Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171322024-02-08T06:46:03.574Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.785555447Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171332024-02-08T06:46:03.574Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.785597944Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171342024-02-08T06:46:03.574Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.785615183Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171352024-02-08T06:46:03.574Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.785630922Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171362024-02-08T06:46:03.574Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.786179431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171372024-02-08T06:46:03.574Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.786216468Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171382024-02-08T06:46:03.574Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.786820403Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171392024-02-08T06:46:03.575Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.786868019Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171402024-02-08T06:46:03.575Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.786908676Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171412024-02-08T06:46:03.575Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.786941314Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171422024-02-08T06:46:03.575Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.786956622Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171432024-02-08T06:46:03.575Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:02.786977321Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171442024-02-08T06:46:04.569Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.785709528Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171452024-02-08T06:46:04.575Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.785982067Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171462024-02-08T06:46:04.576Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.7868861Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171472024-02-08T06:46:04.576Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.786922447Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171482024-02-08T06:46:04.576Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.787429049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171492024-02-08T06:46:04.576Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.787995196Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171502024-02-08T06:46:04.576Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788025014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171512024-02-08T06:46:04.576Z{"msg":"[0]R flush_numbers[0..12]: [267, 260, 256, 268, 267, 267, 268, 264, 264, 260, 260, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788066591Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171522024-02-08T06:46:04.576Z{"msg":"[0]R generation[0..12]: [156, 153, 150, 157, 156, 156, 157, 155, 155, 153, 153, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788089179Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171532024-02-08T06:46:04.576Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788104638Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171542024-02-08T06:46:04.576Z{"msg":"[1]R flush_numbers[0..12]: [267, 260, 256, 268, 267, 267, 268, 264, 264, 260, 260, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788126996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171552024-02-08T06:46:04.576Z{"msg":"[1]R generation[0..12]: [156, 153, 150, 157, 156, 156, 157, 155, 155, 153, 153, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788141935Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171562024-02-08T06:46:04.576Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788163964Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171572024-02-08T06:46:04.576Z{"msg":"[2]R flush_numbers[0..12]: [267, 260, 256, 268, 267, 267, 268, 264, 264, 260, 260, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788178963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171582024-02-08T06:46:04.576Z{"msg":"[2]R generation[0..12]: [156, 153, 150, 157, 156, 156, 157, 155, 155, 153, 153, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788201381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171592024-02-08T06:46:04.576Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.78821623Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171602024-02-08T06:46:04.577Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788237388Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171612024-02-08T06:46:04.577Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
171622024-02-08T06:46:04.577Z,"time":"2024-02-08T06:46:03.788252267Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171632024-02-08T06:46:04.577Z{"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788278265Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171642024-02-08T06:46:04.577Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788293454Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171652024-02-08T06:46:04.577Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788307713Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171662024-02-08T06:46:04.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788329251Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171672024-02-08T06:46:04.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.78834375Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171682024-02-08T06:46:04.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788365018Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171692024-02-08T06:46:04.577Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788379457Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171702024-02-08T06:46:04.577Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788397586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171712024-02-08T06:46:04.577Z{"msg":"5d19a2d4-59f5-4b74-a8bb-39ff5438ab34 is now active with session: efd194b9-e382-4771-bd7d-5031f4fac3ce","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788420784Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171722024-02-08T06:46:04.577Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788440253Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770,"session_id":"efd194b9-e382-4771-bd7d-5031f4fac3ce"}
171732024-02-08T06:46:04.577Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:03.788455862Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770}
171742024-02-08T06:46:04.578ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
171752024-02-08T06:46:04.578ZLoading write count information from file "/tmp/test_repair_verify.data"
171762024-02-08T06:46:04.578ZRun Repair workload
171772024-02-08T06:46:04.578Z01/30 Write block 269 len 4096 data: 13
171782024-02-08T06:46:04.578Z02/30 Read block 100 len 32768
171792024-02-08T06:46:04.583Z03/30 Read block 12 len 36864
171802024-02-08T06:46:04.583Z04/30 Read block 364 len 28672
171812024-02-08T06:46:04.583Z05/30 Read block 376 len 12288
171822024-02-08T06:46:04.583Z06/30 Read block 467 len 20480
171832024-02-08T06:46:04.583Z07/30 Write block 254 len 36864 data: 10 13 11 9 9 12 12 16 16
171842024-02-08T06:46:04.583Z08/30 Read block 296 len 8192
171852024-02-08T06:46:04.583Z09/30 Flush
171862024-02-08T06:46:04.587Z10/30 Write block 455 len 24576 data: 16 14 14 14 13 11
171872024-02-08T06:46:04.587Z11/30 Write block 502 len 40960 data: 6 5 8 6 6 7 7 10 12 9
171882024-02-08T06:46:04.587Z12/30 Flush
171892024-02-08T06:46:04.587Z13/30 Write block 233 len 40960 data: 10 10 12 10 9 12 13 12 15 12
171902024-02-08T06:46:04.591Z14/30 Write block 359 len 20480 data: 12 11 8 9 9
171912024-02-08T06:46:04.591Z15/30 Read block 45 len 4096
171922024-02-08T06:46:04.591Z16/30 Read block 125 len 24576
171932024-02-08T06:46:04.591Z17/30 Write block 267 len 40960 data: 16 16 14 9 7 8 10 10 12 14
171942024-02-08T06:46:04.591Z18/30 Read block 470 len 8192
171952024-02-08T06:46:04.595Z19/30 Read block 107 len 16384
171962024-02-08T06:46:04.595Z20/30 Flush
171972024-02-08T06:46:04.595Z21/30 Write block 468 len 32768 data: 12 12 11 11 13 12 14 13
171982024-02-08T06:46:04.604Z22/30 Read block 308 len 24576
171992024-02-08T06:46:04.604Z23/30 Read block 126 len 28672
172002024-02-08T06:46:04.604Z24/30 Write block 81 len 28672 data: 12 14 13 13 13 13 12
172012024-02-08T06:46:04.604Z25/30 Read block 272 len 32768
172022024-02-08T06:46:04.604Z26/30 Write block 251 len 8192 data: 15 12
172032024-02-08T06:46:04.609Z27/30 Write block 96 len 32768 data: 19 20 18 18 19 16 14 14
172042024-02-08T06:46:04.609Z28/30 Read block 272 len 32768
172052024-02-08T06:46:04.609Z29/30 Write block 199 len 40960 data: 18 16 12 13 12 12 14 14 15 13
172062024-02-08T06:46:04.609Z30/30 Write block 545 len 32768 data: 10 14 18 21 23 23 22 20
172072024-02-08T06:46:04.610Z----------------------------------------------------------------
172082024-02-08T06:46:04.610Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
172092024-02-08T06:46:04.610ZStates: Active Active Active
172102024-02-08T06:46:04.610ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172112024-02-08T06:46:04.610Z 1 Acked 1000 Write 1 Done Done Done false
172122024-02-08T06:46:04.610Z 2 Acked 1001 Read 8 Done Done Done false
172132024-02-08T06:46:04.610Z 3 Acked 1002 Read 9 Sent Done Done false
172142024-02-08T06:46:04.610Z 4 Acked 1003 Read 7 Sent Done Done false
172152024-02-08T06:46:04.610Z 5 Acked 1004 Read 3 Sent Done Done false
172162024-02-08T06:46:04.610Z 6 Acked 1005 Read 5 Sent Done Done false
172172024-02-08T06:46:04.610Z 7 Acked 1006 Write 9 Sent Done Done false
172182024-02-08T06:46:04.610Z 8 Acked 1007 Read 2 Sent Done Done false
172192024-02-08T06:46:04.610Z 9 Acked 1008 Flush 0 Sent Done Done false
172202024-02-08T06:46:04.610Z 10 Acked 1009 Write 6 Sent Done Done false
172212024-02-08T06:46:04.610Z 11 Acked 1010 Write 10 Sent Done Done false
172222024-02-08T06:46:04.611Z 12 Acked 1011 Flush 0 Sent Done Done false
172232024-02-08T06:46:04.611Z 13 Acked 1012 Write 10 Sent Done Done false
172242024-02-08T06:46:04.611Z 14 Acked 1013 Write 5 Sent Done Done false
172252024-02-08T06:46:04.611Z 15 Acked 1014 Read 1 Sent Done Done false
172262024-02-08T06:46:04.611Z 16 Acked 1015 Read 6 Sent Done Done false
172272024-02-08T06:46:04.611Z 17 Acked 1016 Write 10 Sent Done Done false
172282024-02-08T06:46:04.611Z 18 Acked 1017 Read 2 Sent Done Done false
172292024-02-08T06:46:04.611Z 19 Acked 1018 Read 4 Sent Done Done false
172302024-02-08T06:46:04.611Z 20 Acked 1019 Flush 0 Sent Done Done false
172312024-02-08T06:46:04.611Z 21 Acked 1020 Write 8 Sent Done Done false
172322024-02-08T06:46:04.611Z 22 Acked 1021 Read 6 Sent Done Done false
172332024-02-08T06:46:04.611Z 23 Acked 1022 Read 7 Sent Done Done false
172342024-02-08T06:46:04.611Z 24 Acked 1023 Write 7 Sent Done Done false
172352024-02-08T06:46:04.611Z 25 Acked 1024 Read 8 Sent Done Done false
172362024-02-08T06:46:04.611Z 26 Acked 1025 Write 2 Sent Done Done false
172372024-02-08T06:46:04.611Z 27 Acked 1026 Write 8 Sent Done Done false
172382024-02-08T06:46:04.612Z 28 Acked 1027 Read 8 Sent Done Done false
172392024-02-08T06:46:04.612Z 29 Acked 1028 Write 10 Sent Done Sent false
172402024-02-08T06:46:04.612Z 30 Acked 1029 Write 8 Sent Sent Sent false
172412024-02-08T06:46:04.612Z STATES DS:0 DS:1 DS:2 TOTAL
172422024-02-08T06:46:04.612Z New 0 0 0 0
172432024-02-08T06:46:04.612Z Sent 28 1 2 31
172442024-02-08T06:46:04.612Z Done 2 29 28 59
172452024-02-08T06:46:04.612Z Skipped 0 0 0 0
172462024-02-08T06:46:04.612Z Error 0 0 0 0
172472024-02-08T06:46:04.612ZLast Flush: 0 1019 1019
172482024-02-08T06:46:04.612ZDownstairs last five completed:
172492024-02-08T06:46:04.612ZUpstairs last five completed: 30 29 28 27 26
172502024-02-08T06:46:04.612Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:03.821835496Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1770}
172512024-02-08T06:46:04.612ZWrote out file "/tmp/test_repair_verify.data"
172522024-02-08T06:46:04.619Z
172532024-02-08T06:46:04.619ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
172542024-02-08T06:46:04.744ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172552024-02-08T06:46:04.744Z 0 000-019 156 156 156 267 267 267 F F F
172562024-02-08T06:46:04.744Z 1 020-039 153 153 153 260 260 260 F F F
172572024-02-08T06:46:04.744Z 2 040-059 150 150 150 256 256 256 F F F
172582024-02-08T06:46:04.744Z 3 060-079 157 157 157 268 268 268 F F F
172592024-02-08T06:46:04.745Z 4 080-099 156 156 156 267 267 267 F T T <---
172602024-02-08T06:46:04.745Z 5 100-119 156 156 156 267 267 267 F T T <---
172612024-02-08T06:46:04.745Z 6 120-139 157 157 157 268 268 268 F F F
172622024-02-08T06:46:04.745Z 7 140-159 155 155 155 264 264 264 F F F
172632024-02-08T06:46:04.745Z 8 160-179 155 155 155 264 264 264 F F F
172642024-02-08T06:46:04.749Z 9 180-199 153 153 153 260 260 260 F T T <---
172652024-02-08T06:46:04.749Z 10 200-219 153 153 153 260 260 260 F T T <---
172662024-02-08T06:46:04.749Z 11 220-239 154 158 158 263 271 271 F F F <---
172672024-02-08T06:46:04.749Z 12 240-259 148 158 158 253 271 271 F T T <---
172682024-02-08T06:46:04.749Z 13 260-279 150 158 158 255 271 271 T F F <---
172692024-02-08T06:46:04.749Z 14 280-299 148 148 148 253 253 253 F F F
172702024-02-08T06:46:04.749Z 15 300-319 154 154 154 262 262 262 F F F
172712024-02-08T06:46:04.749Z 16 320-339 143 143 143 240 240 240 F F F
172722024-02-08T06:46:04.749Z 17 340-359 157 158 158 268 271 271 F F F <---
172732024-02-08T06:46:04.750Z 18 360-379 155 158 158 264 271 271 F F F <---
172742024-02-08T06:46:04.750Z 19 380-399 154 154 154 263 263 263 F F F
172752024-02-08T06:46:04.750Z 20 400-419 153 153 153 260 260 260 F F F
172762024-02-08T06:46:04.750Z 21 420-439 156 156 156 265 265 265 F F F
172772024-02-08T06:46:04.750Z 22 440-459 156 158 158 265 270 270 F F F <---
172782024-02-08T06:46:04.750Z 23 460-479 155 158 158 264 270 270 F T T <---
172792024-02-08T06:46:04.750Z 24 480-499 153 153 153 260 260 260 F F F
172802024-02-08T06:46:04.750Z 25 500-519 157 158 158 268 270 270 F F F <---
172812024-02-08T06:46:04.750Z 26 520-539 156 156 156 266 266 266 F F F
172822024-02-08T06:46:04.750Z 27 540-559 157 157 157 268 268 268 F T T <---
172832024-02-08T06:46:04.750Z 28 560-579 153 153 153 260 260 260 F F F
172842024-02-08T06:46:04.750Z 29 580-599 154 154 154 262 262 262 F F F
172852024-02-08T06:46:04.750ZMax gen: 158, Max flush: 271
172862024-02-08T06:46:04.750ZError: Difference in extent metadata found!
172872024-02-08T06:46:04.751ZA Difference in extent metadata is expected here
172882024-02-08T06:46:04.751ZOn loop 79
172892024-02-08T06:46:04.751Z
172902024-02-08T06:46:04.751ZVerifying data now
172912024-02-08T06:46:09.283ZNo change after verify
172922024-02-08T06:46:09.284ZLoop: 79 Downstairs dump after verify (and repair):
172932024-02-08T06:46:10.565ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172942024-02-08T06:46:10.565Z 0 000-019 156 156 156 267 267 267 F F F
172952024-02-08T06:46:10.565Z 1 020-039 153 153 153 260 260 260 F F F
172962024-02-08T06:46:10.565Z 2 040-059 150 150 150 256 256 256 F F F
172972024-02-08T06:46:10.565Z 3 060-079 157 157 157 268 268 268 F F F
172982024-02-08T06:46:10.565Z 4 080-099 159 159 159 272 272 272 F F F
172992024-02-08T06:46:10.565Z 5 100-119 159 159 159 272 272 272 F F F
173002024-02-08T06:46:10.565Z 6 120-139 157 157 157 268 268 268 F F F
173012024-02-08T06:46:10.565Z 7 140-159 155 155 155 264 264 264 F F F
173022024-02-08T06:46:10.565Z 8 160-179 155 155 155 264 264 264 F F F
173032024-02-08T06:46:10.565Z 9 180-199 159 159 159 272 272 272 F F F
173042024-02-08T06:46:10.565Z 10 200-219 159 159 159 272 272 272 F F F
173052024-02-08T06:46:10.566Z 11 220-239 158 158 158 271 271 271 F F F
173062024-02-08T06:46:10.566Z 12 240-259 159 159 159 272 272 272 F F F
173072024-02-08T06:46:10.566Z 13 260-279 158 158 158 271 271 271 F F F
173082024-02-08T06:46:10.566Z 14 280-299 148 148 148 253 253 253 F F F
173092024-02-08T06:46:10.566Z 15 300-319 154 154 154 262 262 262 F F F
173102024-02-08T06:46:10.566Z 16 320-339 143 143 143 240 240 240 F F F
173112024-02-08T06:46:10.566Z 17 340-359 158 158 158 271 271 271 F F F
173122024-02-08T06:46:10.566Z 18 360-379 158 158 158 271 271 271 F F F
173132024-02-08T06:46:10.566Z 19 380-399 154 154 154 263 263 263 F F F
173142024-02-08T06:46:10.566Z 20 400-419 153 153 153 260 260 260 F F F
173152024-02-08T06:46:10.566Z 21 420-439 156 156 156 265 265 265 F F F
173162024-02-08T06:46:10.566Z 22 440-459 158 158 158 270 270 270 F F F
173172024-02-08T06:46:10.566Z 23 460-479 159 159 159 272 272 272 F F F
173182024-02-08T06:46:10.566Z 24 480-499 153 153 153 260 260 260 F F F
173192024-02-08T06:46:10.566Z 25 500-519 158 158 158 270 270 270 F F F
173202024-02-08T06:46:10.567Z 26 520-539 156 156 156 266 266 266 F F F
173212024-02-08T06:46:10.567Z 27 540-559 159 159 159 272 272 272 F F F
173222024-02-08T06:46:10.567Z 28 560-579 153 153 153 260 260 260 F F F
173232024-02-08T06:46:10.567Z 29 580-599 154 154 154 262 262 262 F F F
173242024-02-08T06:46:10.567ZMax gen: 159, Max flush: 272
173252024-02-08T06:46:10.567Z
173262024-02-08T06:46:10.567ZBegin loop 80
173272024-02-08T06:46:10.567ZDownstairs to restart: 2
173282024-02-08T06:46:10.567Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531011641Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778}
173292024-02-08T06:46:10.567Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0Crucible runtime is spawned
173302024-02-08T06:46:10.567Z,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531369504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778}
173312024-02-08T06:46:10.567Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.53141788Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778}
173322024-02-08T06:46:10.567Z{"msg":"Crucible db4cb969-553f-4fbf-b840-8773b6fba8af has session id: 53be1d2f-a3d8-4467-866a-dcefcf2a2a01","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531458327Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173332024-02-08T06:46:10.567Z{"msg":"Upstairs opts: Upstairs UUID: db4cb969-553f-4fbf-b840-8773b6fba8af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531492864Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173342024-02-08T06:46:10.568Z{"msg":"Crucible stats registered with UUID: db4cb969-553f-4fbf-b840-8773b6fba8af","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531527942Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173352024-02-08T06:46:10.568Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.53154628Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778}
173362024-02-08T06:46:10.568Z{"msg":"db4cb969-553f-4fbf-b840-8773b6fba8af active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531581748Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173372024-02-08T06:46:10.568Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531609726Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173382024-02-08T06:46:10.568Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531660982Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173392024-02-08T06:46:10.568Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531712578Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173402024-02-08T06:46:10.568Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531748925Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173412024-02-08T06:46:10.568Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531798611Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173422024-02-08T06:46:10.568Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531830219Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173432024-02-08T06:46:10.568Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531861647Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173442024-02-08T06:46:10.568Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531893214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173452024-02-08T06:46:10.568Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.531924742Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173462024-02-08T06:46:10.568Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.53194849Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173472024-02-08T06:46:10.568Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.532025894Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173482024-02-08T06:46:10.568Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.53207985Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173492024-02-08T06:46:10.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.532572283Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173502024-02-08T06:46:10.569Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.53261323Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173512024-02-08T06:46:10.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.532660097Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173522024-02-08T06:46:10.569Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.532703214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173532024-02-08T06:46:10.569Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.533218345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173542024-02-08T06:46:10.569Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.533262021Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173552024-02-08T06:46:10.569Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.533830659Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173562024-02-08T06:46:10.569Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.533878795Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173572024-02-08T06:46:10.569Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.533906873Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173582024-02-08T06:46:10.569Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.53394704Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173592024-02-08T06:46:10.569Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.533971678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173602024-02-08T06:46:10.569Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.534007226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173612024-02-08T06:46:10.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.53407608Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173622024-02-08T06:46:10.569Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.534114188Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173632024-02-08T06:46:10.570Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.534565654Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173642024-02-08T06:46:10.570Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.534992902Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173652024-02-08T06:46:10.570Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535029999Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173662024-02-08T06:46:10.570Z{"msg":"[0]R flush_numbers[0..12]: [267, 260, 256, 268, 272, 272, 268, 264, 264, 272, 272, 271]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535047907Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173672024-02-08T06:46:10.570Z{"msg":"[0]R generation[0..12]: [156, 153, 150, 157, 159, 159, 157, 155, 155, 159, 159, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535080425Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173682024-02-08T06:46:10.570Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535114242Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173692024-02-08T06:46:10.570Z{"msg":"[1]R flush_numbers[0..12]: [267, 260, 256, 268, 272, 272, 268, 264, 264, 272, 272, 271]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.53514807Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173702024-02-08T06:46:10.570ZWait for a query_work_queue command to finish before sending IO
173712024-02-08T06:46:10.570Z{"msg":"[1]R generation[0..12]: [156, 153, 150, 157, 159, 159, 157, 155, 155, 159, 159, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535182937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173722024-02-08T06:46:10.570Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535215515Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173732024-02-08T06:46:10.570Z{"msg":"[2]R flush_numbers[0..12]: [267, 260, 256, 268, 272, 272, 268, 264, 264, 272, 272, 271]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535248672Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173742024-02-08T06:46:10.570Z{"msg":"[2]R generation[0..12]: [156, 153, 150, 157, 159, 159, 157, 155, 155, 159, 159, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.53528123Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173752024-02-08T06:46:10.570Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535314458Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173762024-02-08T06:46:10.570Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535346105Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173772024-02-08T06:46:10.570Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535362654Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173782024-02-08T06:46:10.570Z{"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535392952Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173792024-02-08T06:46:10.571Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535422339Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173802024-02-08T06:46:10.571Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535454917Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173812024-02-08T06:46:10.571Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535484905Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173822024-02-08T06:46:10.571Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535516902Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173832024-02-08T06:46:10.571Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.53554827Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173842024-02-08T06:46:10.571ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
173852024-02-08T06:46:10.571Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535581238Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173862024-02-08T06:46:10.571Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535597956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173872024-02-08T06:46:10.571Z{"msg":"db4cb969-553f-4fbf-b840-8773b6fba8af is now active with session: 53be1d2f-a3d8-4467-866a-dcefcf2a2a01","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535626804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173882024-02-08T06:46:10.571Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535646353Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778,"session_id":"53be1d2f-a3d8-4467-866a-dcefcf2a2a01"}
173892024-02-08T06:46:10.571Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:08.535673431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778}
173902024-02-08T06:46:10.571ZLoading write count information from file "/tmp/test_repair_verify.data"
173912024-02-08T06:46:10.571ZRun Repair workload
173922024-02-08T06:46:10.571Z01/30 Write block 30 len 20480 data: 14 14 12 12 11
173932024-02-08T06:46:10.577Z02/30 Write block 579 len 28672 data: 13 11 11 11 11 11 12
173942024-02-08T06:46:10.577Z03/30 Write block 227 len 8192 data: 17 18
173952024-02-08T06:46:10.577Z04/30 Read block 583 len 40960
173962024-02-08T06:46:10.577Z05/30 Write block 125 len 4096 data: 10
173972024-02-08T06:46:10.577Z06/30 Flush
173982024-02-08T06:46:10.577Z07/30 Write block 484 len 36864 data: 11 12 14 13 15 13 14 14 13
173992024-02-08T06:46:10.577Z08/30 Write block 496 len 28672 data: 8 7 9 10 11 9 7
174002024-02-08T06:46:10.578Z09/30 Write block 421 len 4096 data: 13
174012024-02-08T06:46:10.578Z10/30 Write block 391 len 12288 data: 9 11 9
174022024-02-08T06:46:10.578Z11/30 Flush
174032024-02-08T06:46:10.578Z12/30 Write block 548 len 40960 data: 22 24 24 23 21 19 17 17 20 13
174042024-02-08T06:46:10.578Z13/30 Read block 219 len 20480
174052024-02-08T06:46:10.578Z14/30 Write block 487 len 16384 data: 14 16 14 15
174062024-02-08T06:46:10.578Z15/30 Read block 440 len 32768
174072024-02-08T06:46:10.578Z16/30 Read block 72 len 36864
174082024-02-08T06:46:10.578Z17/30 Read block 365 len 16384
174092024-02-08T06:46:10.578Z18/30 Read block 378 len 40960
174102024-02-08T06:46:10.578Z19/30 Write block 521 len 36864 data: 9 9 10 12 12 12 11 12 12
174112024-02-08T06:46:10.578Z20/30 Write block 438 len 16384 data: 15 11 13 13
174122024-02-08T06:46:10.578Z21/30 Write block 181 len 24576 data: 15 15 11 11 14 13
174132024-02-08T06:46:10.578Z22/30 Write block 357 len 28672 data: 11 10 13 12 9 10 10
174142024-02-08T06:46:10.578Z23/30 Read block 351 len 12288
174152024-02-08T06:46:10.578Z24/30 Read block 180 len 32768
174162024-02-08T06:46:10.579Z25/30 Read block 538 len 32768
174172024-02-08T06:46:10.579Z26/30 Read block 337 len 20480
174182024-02-08T06:46:10.579Z27/30 Write block 269 len 20480 data: 15 10 8 9 11
174192024-02-08T06:46:10.579Z28/30 Read block 425 len 32768
174202024-02-08T06:46:10.579Z29/30 Read block 456 len 16384
174212024-02-08T06:46:10.579Z30/30 Write block 214 len 28672 data: 17 15 16 16 14 16 16
174222024-02-08T06:46:10.579Z----------------------------------------------------------------
174232024-02-08T06:46:10.579Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
174242024-02-08T06:46:10.579ZStates: Active Active Active
174252024-02-08T06:46:10.579ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174262024-02-08T06:46:10.579Z 1 Acked 1000 Write 5 Done Done Sent false
174272024-02-08T06:46:10.579Z 2 Acked 1001 Write 7 Done Done Sent false
174282024-02-08T06:46:10.579Z 3 Acked 1002 Write 2 Done Done Sent false
174292024-02-08T06:46:10.579Z 4 Acked 1003 Read 10 Done Done Sent false
174302024-02-08T06:46:10.579Z 5 Acked 1004 Write 1 Done Done Sent false
174312024-02-08T06:46:10.580Z 6 Acked 1005 Flush 0 Done Done Sent false
174322024-02-08T06:46:10.580Z 7 Acked 1006 Write 9 Done Done Sent false
174332024-02-08T06:46:10.580Z 8 Acked 1007 Write 7 Done Done Sent false
174342024-02-08T06:46:10.580Z 9 Acked 1008 Write 1 Done Done Sent false
174352024-02-08T06:46:10.580Z 10 Acked 1009 Write 3 Done Done Sent false
174362024-02-08T06:46:10.580Z 11 Acked 1010 Flush 0 Done Done Sent false
174372024-02-08T06:46:10.580Z 12 Acked 1011 Write 10 Done Done Sent false
174382024-02-08T06:46:10.580Z 13 Acked 1012 Read 5 Done Done Sent false
174392024-02-08T06:46:10.580Z 14 Acked 1013 Write 4 Done Done Sent false
174402024-02-08T06:46:10.580Z 15 Acked 1014 Read 8 Done Done Sent false
174412024-02-08T06:46:10.580Z 16 Acked 1015 Read 9 Done Done Sent false
174422024-02-08T06:46:10.580Z 17 Acked 1016 Read 4 Done Done Sent false
174432024-02-08T06:46:10.580Z 18 Acked 1017 Read 10 Done Done Sent false
174442024-02-08T06:46:10.580Z 19 Acked 1018 Write 9 Done Done Sent false
174452024-02-08T06:46:10.580Z 20 Acked 1019 Write 4 Done Done Sent false
174462024-02-08T06:46:10.580Z 21 Acked 1020 Write 6 Done Done Sent false
174472024-02-08T06:46:10.581Z 22 Acked 1021 Write 7 Done Done Sent false
174482024-02-08T06:46:10.581Z 23 Acked 1022 Read 3 Done Done Sent false
174492024-02-08T06:46:10.581Z 24 Acked 1023 Read 8 Done Done Sent false
174502024-02-08T06:46:10.581Z 25 Acked 1024 Read 8 Done Done Sent false
174512024-02-08T06:46:10.581Z 26 Acked 1025 Read 5 Done Done Sent false
174522024-02-08T06:46:10.581Z 27 Acked 1026 Write 5 Done Done Sent false
174532024-02-08T06:46:10.581Z 28 Acked 1027 Read 8 Done Done Sent false
174542024-02-08T06:46:10.581Z 29 Acked 1028 Read 4 Done Done Sent false
174552024-02-08T06:46:10.581Z 30 Acked 1029 Write 7 Sent Sent Sent false
174562024-02-08T06:46:10.581Z STATES DS:0 DS:1 DS:2 TOTAL
174572024-02-08T06:46:10.581Z New 0 0 0 0
174582024-02-08T06:46:10.581Z Sent 1 1 30 32
174592024-02-08T06:46:10.581Z Done 29 29 0 58
174602024-02-08T06:46:10.581Z Skipped 0 0 0 0
174612024-02-08T06:46:10.581Z Error 0 0 0 0
174622024-02-08T06:46:10.581ZLast Flush: 1010 1010 0
174632024-02-08T06:46:10.581ZDownstairs last five completed:
174642024-02-08T06:46:10.582ZUpstairs last five completed: 30 29 28 27 26
174652024-02-08T06:46:10.582Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:08.565743182Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1778}
174662024-02-08T06:46:10.582ZWrote out file "/tmp/test_repair_verify.data"
174672024-02-08T06:46:10.582Z
174682024-02-08T06:46:10.582ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
174692024-02-08T06:46:10.582ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174702024-02-08T06:46:10.582Z 0 000-019 156 156 156 267 267 267 F F F
174712024-02-08T06:46:10.582Z 1 020-039 160 160 153 273 273 260 F F F <---
174722024-02-08T06:46:10.582Z 2 040-059 150 150 150 256 256 256 F F F
174732024-02-08T06:46:10.582Z 3 060-079 157 157 157 268 268 268 F F F
174742024-02-08T06:46:10.582Z 4 080-099 159 159 159 272 272 272 F F F
174752024-02-08T06:46:10.582Z 5 100-119 159 159 159 272 272 272 F F F
174762024-02-08T06:46:10.582Z 6 120-139 160 160 157 273 273 268 F F F <---
174772024-02-08T06:46:10.582Z 7 140-159 155 155 155 264 264 264 F F F
174782024-02-08T06:46:10.582Z 8 160-179 155 155 155 264 264 264 F F F
174792024-02-08T06:46:10.582Z 9 180-199 159 159 159 272 272 272 T T F <---
174802024-02-08T06:46:10.582Z 10 200-219 159 159 159 272 272 272 T T F <---
174812024-02-08T06:46:10.582Z 11 220-239 160 160 158 273 273 271 T T F <---
174822024-02-08T06:46:10.583Z 12 240-259 159 159 159 272 272 272 F F F
174832024-02-08T06:46:10.583Z 13 260-279 158 158 158 271 271 271 T T F <---
174842024-02-08T06:46:10.583Z 14 280-299 148 148 148 253 253 253 F F F
174852024-02-08T06:46:10.583Z 15 300-319 154 154 154 262 262 262 F F F
174862024-02-08T06:46:10.583Z 16 320-339 143 143 143 240 240 240 F F F
174872024-02-08T06:46:10.583Z 17 340-359 158 158 158 271 271 271 T T F <---
174882024-02-08T06:46:10.583Z 18 360-379 158 158 158 271 271 271 T T F <---
174892024-02-08T06:46:10.583Z 19 380-399 160 160 154 274 274 263 F F F <---
174902024-02-08T06:46:10.583Z 20 400-419 153 153 153 260 260 260 F F F
174912024-02-08T06:46:10.583Z 21 420-439 160 160 156 274 274 265 T T F <---
174922024-02-08T06:46:10.583Z 22 440-459 158 158 158 270 270 270 T T F <---
174932024-02-08T06:46:10.587Z 23 460-479 159 159 159 272 272 272 F F F
174942024-02-08T06:46:10.587Z 24 480-499 160 160 153 274 274 260 T T F <---
174952024-02-08T06:46:10.587Z 25 500-519 160 160 158 274 274 270 F F F <---
174962024-02-08T06:46:10.587Z 26 520-539 156 156 156 266 266 266 T T F <---
174972024-02-08T06:46:10.587Z 27 540-559 159 159 159 272 272 272 T T F <---
174982024-02-08T06:46:10.587Z 28 560-579 160 160 153 273 273 260 F F F <---
174992024-02-08T06:46:10.588Z 29 580-599 160 160 154 273 273 262 F F F <---
175002024-02-08T06:46:10.588ZMax gen: 160, Max flush: 274
175012024-02-08T06:46:10.588ZError: Difference in extent metadata found!
175022024-02-08T06:46:10.588ZA Difference in extent metadata is expected here
175032024-02-08T06:46:10.588ZOn loop 80
175042024-02-08T06:46:10.588Z
175052024-02-08T06:46:10.588ZVerifying data now
175062024-02-08T06:46:14.189ZNo change after verify
175072024-02-08T06:46:14.189ZLoop: 80 Downstairs dump after verify (and repair):
175082024-02-08T06:46:14.204ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175092024-02-08T06:46:14.204Z 0 000-019 156 156 156 267 267 267 F F F
175102024-02-08T06:46:14.204Z 1 020-039 160 160 160 273 273 273 F F F
175112024-02-08T06:46:14.204Z 2 040-059 150 150 150 256 256 256 F F F
175122024-02-08T06:46:14.204Z 3 060-079 157 157 157 268 268 268 F F F
175132024-02-08T06:46:14.204Z 4 080-099 159 159 159 272 272 272 F F F
175142024-02-08T06:46:14.204Z 5 100-119 159 159 159 272 272 272 F F F
175152024-02-08T06:46:14.205Z 6 120-139 160 160 160 273 273 273 F F F
175162024-02-08T06:46:14.205Z 7 140-159 155 155 155 264 264 264 F F F
175172024-02-08T06:46:14.205Z 8 160-179 155 155 155 264 264 264 F F F
175182024-02-08T06:46:14.208Z 9 180-199 161 161 161 275 275 275 F F F
175192024-02-08T06:46:14.208Z 10 200-219 161 161 161 275 275 275 F F F
175202024-02-08T06:46:14.208Z 11 220-239 161 161 161 275 275 275 F F F
175212024-02-08T06:46:14.208Z 12 240-259 159 159 159 272 272 272 F F F
175222024-02-08T06:46:14.208Z 13 260-279 161 161 161 275 275 275 F F F
175232024-02-08T06:46:14.209Z 14 280-299 148 148 148 253 253 253 F F F
175242024-02-08T06:46:14.209Z 15 300-319 154 154 154 262 262 262 F F F
175252024-02-08T06:46:14.209Z 16 320-339 143 143 143 240 240 240 F F F
175262024-02-08T06:46:14.209Z 17 340-359 161 161 161 275 275 275 F F F
175272024-02-08T06:46:14.209Z 18 360-379 161 161 161 275 275 275 F F F
175282024-02-08T06:46:14.209Z 19 380-399 160 160 160 274 274 274 F F F
175292024-02-08T06:46:14.209Z 20 400-419 153 153 153 260 260 260 F F F
175302024-02-08T06:46:14.209Z 21 420-439 161 161 161 275 275 275 F F F
175312024-02-08T06:46:14.209Z 22 440-459 161 161 161 275 275 275 F F F
175322024-02-08T06:46:14.209Z 23 460-479 159 159 159 272 272 272 F F F
175332024-02-08T06:46:14.209Z 24 480-499 161 161 161 275 275 275 F F F
175342024-02-08T06:46:14.209Z 25 500-519 160 160 160 274 274 274 F F F
175352024-02-08T06:46:14.209Z 26 520-539 161 161 161 275 275 275 F F F
175362024-02-08T06:46:14.209Z 27 540-559 161 161 161 275 275 275 F F F
175372024-02-08T06:46:14.210Z 28 560-579 160 160 160 273 273 273 F F F
175382024-02-08T06:46:14.210Z 29 580-599 160 160 160 273 273 273 F F F
175392024-02-08T06:46:14.210ZMax gen: 161, Max flush: 275
175402024-02-08T06:46:14.210Z
175412024-02-08T06:46:14.210ZBegin loop 81
175422024-02-08T06:46:14.210ZDownstairs to restart: 1
175432024-02-08T06:46:14.227Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443187154Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786}
175442024-02-08T06:46:14.227Z{"Crucible runtime is spawned
175452024-02-08T06:46:14.233Zmsg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.44350383Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786}
175462024-02-08T06:46:14.233Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443592834Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786}
175472024-02-08T06:46:14.233Z{"msg":"Crucible 69ed854f-d745-428d-8119-65872051eb01 has session id: d2370657-a344-4852-b5c6-c9b8ab7e3e21","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443631131Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175482024-02-08T06:46:14.233Z{"msg":"Upstairs opts: Upstairs UUID: 69ed854f-d745-428d-8119-65872051eb01, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443674337Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175492024-02-08T06:46:14.233Z{"msg":"Crucible stats registered with UUID: 69ed854f-d745-428d-8119-65872051eb01","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443710925Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175502024-02-08T06:46:14.233Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443745612Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786}
175512024-02-08T06:46:14.233Z{"msg":"69ed854f-d745-428d-8119-65872051eb01 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.44377573Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175522024-02-08T06:46:14.234Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443813057Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175532024-02-08T06:46:14.234Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443854324Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175542024-02-08T06:46:14.234Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443885652Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175552024-02-08T06:46:14.234Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443942077Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175562024-02-08T06:46:14.234Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.443989794Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175572024-02-08T06:46:14.234Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.444137013Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175582024-02-08T06:46:14.234Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.444204048Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"io task","client":"0","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175592024-02-08T06:46:14.234Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.444234195Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"io task","client":"2","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175602024-02-08T06:46:14.234Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.444250564Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175612024-02-08T06:46:14.234Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.444265623Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"io task","client":"2","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175622024-02-08T06:46:14.234Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.444292961Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"io task","client":"0","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175632024-02-08T06:46:14.234Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:13.444322949Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175642024-02-08T06:46:14.234Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:13.444365046Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175652024-02-08T06:46:14.234Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:13.444382384Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175662024-02-08T06:46:14.235Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.444395294Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175672024-02-08T06:46:14.235Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.444417322Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175682024-02-08T06:46:14.235Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.444544852Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175692024-02-08T06:46:14.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.44498069Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175702024-02-08T06:46:14.235Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.445008847Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175712024-02-08T06:46:14.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.445025256Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175722024-02-08T06:46:14.235Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.445039805Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175732024-02-08T06:46:14.235Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.445544487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175742024-02-08T06:46:14.235Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.445661488Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175752024-02-08T06:46:14.235Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.446077107Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175762024-02-08T06:46:14.235Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.446145302Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175772024-02-08T06:46:14.235Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.44617562Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175782024-02-08T06:46:14.235Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.446205747Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175792024-02-08T06:46:14.235Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.446231825Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175802024-02-08T06:46:14.235Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:13.446263673Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175812024-02-08T06:46:15.230Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.445120901Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175822024-02-08T06:46:15.235Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.445341235Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175832024-02-08T06:46:15.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.446259146Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175842024-02-08T06:46:15.235Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.446295083Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175852024-02-08T06:46:15.235Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.446792096Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175862024-02-08T06:46:15.235Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447348354Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175872024-02-08T06:46:15.235Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.4473947Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175882024-02-08T06:46:15.235Z{"msg":"[0]R flush_numbers[0..12]: [267, 273, 256, 268, 272, 272, 273, 264, 264, 275, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447444527Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175892024-02-08T06:46:15.235Z{"msg":"[0]R generation[0..12]: [156, 160, 150, 157, 159, 159, 160, 155, 155, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447505012Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175902024-02-08T06:46:15.235Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
175912024-02-08T06:46:15.235Z,"time":"2024-02-08T06:46:14.44754134Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175922024-02-08T06:46:15.235Z{"msg":"[1]R flush_numbers[0..12]: [267, 273, 256, 268, 272, 272, 273, 264, 264, 275, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447560748Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175932024-02-08T06:46:15.235Z{"msg":"[1]R generation[0..12]: [156, 160, 150, 157, 159, 159, 160, 155, 155, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447582546Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175942024-02-08T06:46:15.236Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447605015Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175952024-02-08T06:46:15.236Z{"msg":"[2]R flush_numbers[0..12]: [267, 273, 256, 268, 272, 272, 273, 264, 264, 275, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447628643Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175962024-02-08T06:46:15.236Z{"msg":"[2]R generation[0..12]: [156, 160, 150, 157, 159, 159, 160, 155, 155, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447653971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175972024-02-08T06:46:15.236Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.4476692Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175982024-02-08T06:46:15.236Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447683709Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
175992024-02-08T06:46:15.236Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447705547Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176002024-02-08T06:46:15.236Z{"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447736685Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176012024-02-08T06:46:15.236Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447751814Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176022024-02-08T06:46:15.236Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447765333Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176032024-02-08T06:46:15.236Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447784611Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176042024-02-08T06:46:15.236Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.44779873Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176052024-02-08T06:46:15.236Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447830618Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176062024-02-08T06:46:15.236Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447869465Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176072024-02-08T06:46:15.236Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447904032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176082024-02-08T06:46:15.236Z{"msg":"69ed854f-d745-428d-8119-65872051eb01 is now active with session: d2370657-a344-4852-b5c6-c9b8ab7e3e21","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447942509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176092024-02-08T06:46:15.237Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:14.447976977Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786,"session_id":"d2370657-a344-4852-b5c6-c9b8ab7e3e21"}
176102024-02-08T06:46:15.237Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176112024-02-08T06:46:15.237Z,"time":"2024-02-08T06:46:14.448028773Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786}
176122024-02-08T06:46:15.237ZLoading write count information from file "/tmp/test_repair_verify.data"
176132024-02-08T06:46:15.237ZRun Repair workload
176142024-02-08T06:46:15.237Z01/30 Write block 283 len 36864 data: 9 11 10 9 10 12 10 14 16
176152024-02-08T06:46:15.240Z02/30 Write block 456 len 40960 data: 15 15 15 14 12 12 15 15 11 10
176162024-02-08T06:46:15.240Z03/30 Write block 382 len 12288 data: 10 9 13
176172024-02-08T06:46:15.240Z04/30 Read block 142 len 32768
176182024-02-08T06:46:15.241Z05/30 Write block 494 len 4096 data: 12
176192024-02-08T06:46:15.241Z06/30 Read block 198 len 4096
176202024-02-08T06:46:15.241Z07/30 Flush
176212024-02-08T06:46:15.243Z08/30 Write block 198 len 40960 data: 17 19 17 13 14 13 13 15 15 16
176222024-02-08T06:46:15.243Z09/30 Write block 39 len 16384 data: 10 9 9 9
176232024-02-08T06:46:15.244Z10/30 Write block 358 len 24576 data: 11 14 13 10 11 11
176242024-02-08T06:46:15.247Z11/30 Write block 159 len 36864 data: 15 13 14 16 16 16 18 19 21
176252024-02-08T06:46:15.247Z12/30 Write block 258 len 16384 data: 10 13 13 17
176262024-02-08T06:46:15.247Z13/30 Flush
176272024-02-08T06:46:15.250Z14/30 Write block 471 len 28672 data: 12 14 13 15 14 9 8
176282024-02-08T06:46:15.253Z15/30 Write block 333 len 40960 data: 14 17 16 18 17 18 18 16 16 15
176292024-02-08T06:46:15.253Z16/30 Read block 41 len 32768
176302024-02-08T06:46:15.253Z17/30 Write block 48 len 16384 data: 7 6 9 10
176312024-02-08T06:46:15.256Z18/30 Write block 182 len 32768 data: 16 12 12 15 14 12 12 11
176322024-02-08T06:46:15.256Z19/30 Read block 67 len 12288
176332024-02-08T06:46:15.256Z20/30 Write block 26 len 32768 data: 17 17 16 14 15 15 13 13
176342024-02-08T06:46:15.259Z21/30 Read block 290 len 4096
176352024-02-08T06:46:15.259Z22/30 Write block 544 len 40960 data: 10 11 15 19 23 25 25 24 22 20
176362024-02-08T06:46:15.259Z23/30 Read block 301 len 36864
176372024-02-08T06:46:15.262Z24/30 Write block 241 len 36864 data: 16 13 16 13 13 13 10 11 15
176382024-02-08T06:46:15.262Z25/30 Write block 479 len 4096 data: 8
176392024-02-08T06:46:15.263Z26/30 Read block 82 len 8192
176402024-02-08T06:46:15.263Z27/30 Write block 453 len 40960 data: 13 13 17 16 16 16 15 13 13 16
176412024-02-08T06:46:15.263Z28/30 Read block 347 len 20480
176422024-02-08T06:46:15.267Z29/30 Write block 236 len 16384 data: 11 10 13 14
176432024-02-08T06:46:15.267Z30/30 Read block 495 len 8192
176442024-02-08T06:46:15.267Z----------------------------------------------------------------
176452024-02-08T06:46:15.267Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
176462024-02-08T06:46:15.267ZStates: Active Active Active
176472024-02-08T06:46:15.267ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
176482024-02-08T06:46:15.267Z 1 Acked 1000 Write 9 Done Done Done false
176492024-02-08T06:46:15.267Z 2 Acked 1001 Write 10 Done Done Done false
176502024-02-08T06:46:15.267Z 3 Acked 1002 Write 3 Done Sent Done false
176512024-02-08T06:46:15.267Z 4 Acked 1003 Read 8 Done Sent Done false
176522024-02-08T06:46:15.267Z 5 Acked 1004 Write 1 Done Sent Done false
176532024-02-08T06:46:15.268Z 6 Acked 1005 Read 1 Done Sent Done false
176542024-02-08T06:46:15.268Z 7 Acked 1006 Flush 0 Done Sent Done false
176552024-02-08T06:46:15.268Z 8 Acked 1007 Write 10 Done Sent Done false
176562024-02-08T06:46:15.268Z 9 Acked 1008 Write 4 Done Sent Done false
176572024-02-08T06:46:15.268Z 10 Acked 1009 Write 6 Done Sent Done false
176582024-02-08T06:46:15.268Z 11 Acked 1010 Write 9 Done Sent Done false
176592024-02-08T06:46:15.268Z 12 Acked 1011 Write 4 Done Sent Done false
176602024-02-08T06:46:15.268Z 13 Acked 1012 Flush 0 Done Sent Done false
176612024-02-08T06:46:15.268Z 14 Acked 1013 Write 7 Done Sent Done false
176622024-02-08T06:46:15.268Z 15 Acked 1014 Write 10 Done Sent Done false
176632024-02-08T06:46:15.268Z 16 Acked 1015 Read 8 Done Sent Done false
176642024-02-08T06:46:15.268Z 17 Acked 1016 Write 4 Done Sent Done false
176652024-02-08T06:46:15.268Z 18 Acked 1017 Write 8 Done Sent Done false
176662024-02-08T06:46:15.268Z 19 Acked 1018 Read 3 Done Sent Done false
176672024-02-08T06:46:15.268Z 20 Acked 1019 Write 8 Done Sent Done false
176682024-02-08T06:46:15.269Z 21 Acked 1020 Read 1 Done Sent Done false
176692024-02-08T06:46:15.269Z 22 Acked 1021 Write 10 Done Sent Done false
176702024-02-08T06:46:15.269Z 23 Acked 1022 Read 9 Done Sent Done false
176712024-02-08T06:46:15.269Z 24 Acked 1023 Write 9 Done Sent Done false
176722024-02-08T06:46:15.269Z 25 Acked 1024 Write 1 Done Sent Done false
176732024-02-08T06:46:15.269Z 26 Acked 1025 Read 2 Done Sent Done false
176742024-02-08T06:46:15.269Z 27 Acked 1026 Write 10 Done Sent Done false
176752024-02-08T06:46:15.269Z 28 Acked 1027 Read 5 Done Sent Done false
176762024-02-08T06:46:15.269Z 29 Acked 1028 Write 4 Done Sent Done false
176772024-02-08T06:46:15.269Z 30 Acked 1029 Read 2 Done Sent Sent false
176782024-02-08T06:46:15.269Z STATES DS:0 DS:1 DS:2 TOTAL
176792024-02-08T06:46:15.269Z New 0 0 0 0
176802024-02-08T06:46:15.269Z Sent 0 28 1 29
176812024-02-08T06:46:15.269Z Done 30 2 29 61
176822024-02-08T06:46:15.269Z Skipped 0 0 0 0
176832024-02-08T06:46:15.270Z Error 0 0 0 0
176842024-02-08T06:46:15.270ZLast Flush: 1012 0 1012
176852024-02-08T06:46:15.270ZDownstairs last five completed:
176862024-02-08T06:46:15.270ZUpstairs last five completed: 30 29 28 27 26
176872024-02-08T06:46:15.270Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:14.481531768Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1786}
176882024-02-08T06:46:15.270ZWrote out file "/tmp/test_repair_verify.data"
176892024-02-08T06:46:15.273Z
176902024-02-08T06:46:15.279ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
176912024-02-08T06:46:15.293ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176922024-02-08T06:46:15.293Z 0 000-019 156 156 156 267 267 267 F F F
176932024-02-08T06:46:15.293Z 1 020-039 162 160 162 277 273 277 T F T <---
176942024-02-08T06:46:15.293Z 2 040-059 162 150 162 277 256 277 T F T <---
176952024-02-08T06:46:15.293Z 3 060-079 157 157 157 268 268 268 F F F
176962024-02-08T06:46:15.293Z 4 080-099 159 159 159 272 272 272 F F F
176972024-02-08T06:46:15.293Z 5 100-119 159 159 159 272 272 272 F F F
176982024-02-08T06:46:15.293Z 6 120-139 160 160 160 273 273 273 F F F
176992024-02-08T06:46:15.293Z 7 140-159 162 155 162 277 264 277 F F F <---
177002024-02-08T06:46:15.297Z 8 160-179 162 155 162 277 264 277 F F F <---
177012024-02-08T06:46:15.297Z 9 180-199 162 161 162 277 275 277 T F T <---
177022024-02-08T06:46:15.297Z 10 200-219 162 161 162 277 275 277 F F F <---
177032024-02-08T06:46:15.297Z 11 220-239 161 161 161 275 275 275 T F T <---
177042024-02-08T06:46:15.297Z 12 240-259 162 159 162 277 272 277 T F T <---
177052024-02-08T06:46:15.297Z 13 260-279 162 161 162 277 275 277 F F F <---
177062024-02-08T06:46:15.297Z 14 280-299 162 148 162 276 253 276 F T F <---
177072024-02-08T06:46:15.297Z 15 300-319 154 154 154 262 262 262 F F F
177082024-02-08T06:46:15.297Z 16 320-339 143 143 143 240 240 240 T F T <---
177092024-02-08T06:46:15.297Z 17 340-359 162 161 162 277 275 277 T F T <---
177102024-02-08T06:46:15.297Z 18 360-379 162 161 162 277 275 277 F F F <---
177112024-02-08T06:46:15.297Z 19 380-399 162 160 162 276 274 276 F F F <---
177122024-02-08T06:46:15.297Z 20 400-419 153 153 153 260 260 260 F F F
177132024-02-08T06:46:15.297Z 21 420-439 161 161 161 275 275 275 F F F
177142024-02-08T06:46:15.298Z 22 440-459 162 161 162 276 275 276 T T T <---
177152024-02-08T06:46:15.298Z 23 460-479 162 159 162 276 272 276 T T T <---
177162024-02-08T06:46:15.298Z 24 480-499 162 161 162 276 275 276 F F F <---
177172024-02-08T06:46:15.298Z 25 500-519 160 160 160 274 274 274 F F F
177182024-02-08T06:46:15.298Z 26 520-539 161 161 161 275 275 275 F F F
177192024-02-08T06:46:15.298Z 27 540-559 161 161 161 275 275 275 T F T <---
177202024-02-08T06:46:15.298Z 28 560-579 160 160 160 273 273 273 F F F
177212024-02-08T06:46:15.298Z 29 580-599 160 160 160 273 273 273 F F F
177222024-02-08T06:46:15.298ZMax gen: 162, Max flush: 277
177232024-02-08T06:46:15.298ZError: Difference in extent metadata found!
177242024-02-08T06:46:15.298ZA Difference in extent metadata is expected here
177252024-02-08T06:46:15.298ZOn loop 81
177262024-02-08T06:46:15.298Z
177272024-02-08T06:46:15.301ZVerifying data now
177282024-02-08T06:46:21.132ZNo change after verify
177292024-02-08T06:46:21.132ZLoop: 81 Downstairs dump after verify (and repair):
177302024-02-08T06:46:21.147ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177312024-02-08T06:46:21.147Z 0 000-019 156 156 156 267 267 267 F F F
177322024-02-08T06:46:21.147Z 1 020-039 163 163 163 278 278 278 F F F
177332024-02-08T06:46:21.147Z 2 040-059 163 163 163 278 278 278 F F F
177342024-02-08T06:46:21.147Z 3 060-079 157 157 157 268 268 268 F F F
177352024-02-08T06:46:21.147Z 4 080-099 159 159 159 272 272 272 F F F
177362024-02-08T06:46:21.147Z 5 100-119 159 159 159 272 272 272 F F F
177372024-02-08T06:46:21.147Z 6 120-139 160 160 160 273 273 273 F F F
177382024-02-08T06:46:21.147Z 7 140-159 162 162 162 277 277 277 F F F
177392024-02-08T06:46:21.147Z 8 160-179 162 162 162 277 277 277 F F F
177402024-02-08T06:46:21.147Z 9 180-199 163 163 163 278 278 278 F F F
177412024-02-08T06:46:21.151Z 10 200-219 162 162 162 277 277 277 F F F
177422024-02-08T06:46:21.151Z 11 220-239 163 163 163 278 278 278 F F F
177432024-02-08T06:46:21.151Z 12 240-259 163 163 163 278 278 278 F F F
177442024-02-08T06:46:21.151Z 13 260-279 162 162 162 277 277 277 F F F
177452024-02-08T06:46:21.151Z 14 280-299 162 162 162 276 276 276 F F F
177462024-02-08T06:46:21.151Z 15 300-319 154 154 154 262 262 262 F F F
177472024-02-08T06:46:21.151Z 16 320-339 163 163 163 278 278 278 F F F
177482024-02-08T06:46:21.151Z 17 340-359 163 163 163 278 278 278 F F F
177492024-02-08T06:46:21.151Z 18 360-379 162 162 162 277 277 277 F F F
177502024-02-08T06:46:21.152Z 19 380-399 162 162 162 276 276 276 F F F
177512024-02-08T06:46:21.152Z 20 400-419 153 153 153 260 260 260 F F F
177522024-02-08T06:46:21.152Z 21 420-439 161 161 161 275 275 275 F F F
177532024-02-08T06:46:21.152Z 22 440-459 163 163 163 278 278 278 F F F
177542024-02-08T06:46:21.152Z 23 460-479 163 163 163 278 278 278 F F F
177552024-02-08T06:46:21.152Z 24 480-499 162 162 162 276 276 276 F F F
177562024-02-08T06:46:21.152Z 25 500-519 160 160 160 274 274 274 F F F
177572024-02-08T06:46:21.152Z 26 520-539 161 161 161 275 275 275 F F F
177582024-02-08T06:46:21.152Z 27 540-559 163 163 163 278 278 278 F F F
177592024-02-08T06:46:21.152Z 28 560-579 160 160 160 273 273 273 F F F
177602024-02-08T06:46:21.152Z 29 580-599 160 160 160 273 273 273 F F F
177612024-02-08T06:46:21.152ZMax gen: 163, Max flush: 278
177622024-02-08T06:46:21.152Z
177632024-02-08T06:46:21.152ZBegin loop 82
177642024-02-08T06:46:21.152ZDownstairs to restart: 0
177652024-02-08T06:46:21.168Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383048064Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794}
177662024-02-08T06:46:21.168Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
177672024-02-08T06:46:21.168Z,"time":"2024-02-08T06:46:20.383292916Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794}
177682024-02-08T06:46:21.173Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383331023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794}
177692024-02-08T06:46:21.173Z{"msg":"Crucible 41c5785a-a7e3-48b0-a4ae-a1edb56a0e9b has session id: 946dd6b8-f7c5-4786-b903-5b9b7b2e23e5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.38336472Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177702024-02-08T06:46:21.173Z{"msg":"Upstairs opts: Upstairs UUID: 41c5785a-a7e3-48b0-a4ae-a1edb56a0e9b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383398108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177712024-02-08T06:46:21.173Z{"msg":"Crucible stats registered with UUID: 41c5785a-a7e3-48b0-a4ae-a1edb56a0e9b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383455684Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177722024-02-08T06:46:21.173Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383477842Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794}
177732024-02-08T06:46:21.173Z{"msg":"41c5785a-a7e3-48b0-a4ae-a1edb56a0e9b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383554346Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177742024-02-08T06:46:21.173Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383579214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177752024-02-08T06:46:21.173Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383613252Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177762024-02-08T06:46:21.173Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383628461Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177772024-02-08T06:46:21.173Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383672917Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177782024-02-08T06:46:21.173Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383699865Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177792024-02-08T06:46:21.173Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383727243Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177802024-02-08T06:46:21.174Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383757851Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177812024-02-08T06:46:21.174Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383782919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177822024-02-08T06:46:21.174Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383807867Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"io task","client":"2","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177832024-02-08T06:46:21.174Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:20.383911149Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177842024-02-08T06:46:21.174Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.383950596Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"io task","client":"2","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177852024-02-08T06:46:21.174Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:20.384022211Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177862024-02-08T06:46:21.174Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.384074567Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177872024-02-08T06:46:21.174Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:20.384119304Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177882024-02-08T06:46:21.174Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.384144712Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177892024-02-08T06:46:21.174Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.384160371Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177902024-02-08T06:46:21.174Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.38429918Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177912024-02-08T06:46:21.174Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.384754866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177922024-02-08T06:46:21.174Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.3848379Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177932024-02-08T06:46:21.174Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.384878917Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177942024-02-08T06:46:21.174Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.384894965Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177952024-02-08T06:46:21.174Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.385322193Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177962024-02-08T06:46:21.175Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.385462753Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177972024-02-08T06:46:21.175Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.385874102Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177982024-02-08T06:46:21.175Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.385912189Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
177992024-02-08T06:46:21.175Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.385929818Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178002024-02-08T06:46:21.175Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.385950956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178012024-02-08T06:46:21.175Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.385965795Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178022024-02-08T06:46:21.175Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:20.385988514Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178032024-02-08T06:46:22.171Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.385696568Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178042024-02-08T06:46:22.176Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.385907012Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178052024-02-08T06:46:22.176Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.386764818Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178062024-02-08T06:46:22.176Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.386813484Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178072024-02-08T06:46:22.177Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.387343084Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178082024-02-08T06:46:22.177Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.387876094Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178092024-02-08T06:46:22.177Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.387919461Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178102024-02-08T06:46:22.177Z{"msg":"[0]R flush_numbers[0..12]: [267, 278, 278, 268, 272, 272, 273, 277, 277, 278, 277, 278]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.387940789Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178112024-02-08T06:46:22.177Z{"msg":"[0]R generation[0..12]: [156, 163, 163, 157, 159, 159, 160, 162, 162, 163, 162, 163]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.387968037Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178122024-02-08T06:46:22.177Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.387987246Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178132024-02-08T06:46:22.177Z{"msg":"[1]R flush_numbers[0..12]: [267, 278, 278, 268, 272, 272, 273, 277, 277, 278, 277, 278]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388014114Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178142024-02-08T06:46:22.177Z{"msg":"[1]R generation[0..12]: [156, 163, 163, 157, 159, 159, 160, 162, 162, 163, 162, 163]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388033222Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178152024-02-08T06:46:22.177Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.38805994Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178162024-02-08T06:46:22.177Z{"msg":"[2]R flush_numbers[0..12]: [267, 278, 278, 268, 272, 272, 273, 277, 277, 278, 277, 278]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388076329Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178172024-02-08T06:46:22.177Z{"msg":"[2]R generation[0..12]: [156, 163, 163, 157, 159, 159, 160, 162, 162, 163, 162, 163]","v":0Wait for a query_work_queue command to finish before sending IO
178182024-02-08T06:46:22.177Z,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388100907Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178192024-02-08T06:46:22.177Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388115956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178202024-02-08T06:46:22.177Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388130245Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178212024-02-08T06:46:22.177Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388152214Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178222024-02-08T06:46:22.178Z{"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388166013Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178232024-02-08T06:46:22.178Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388180131Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178242024-02-08T06:46:22.178Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.38820062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178252024-02-08T06:46:22.178Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388213029Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178262024-02-08T06:46:22.178Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388226478Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178272024-02-08T06:46:22.178Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388250686Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178282024-02-08T06:46:22.178Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388269305Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178292024-02-08T06:46:22.178Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388291353Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178302024-02-08T06:46:22.178Z{"msg":"41c5785a-a7e3-48b0-a4ae-a1edb56a0e9b is now active with session: 946dd6b8-f7c5-4786-b903-5b9b7b2e23e5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388305152Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178312024-02-08T06:46:22.178Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.388317251Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794,"session_id":"946dd6b8-f7c5-4786-b903-5b9b7b2e23e5"}
178322024-02-08T06:46:22.178Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:21.38833682Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794}
178332024-02-08T06:46:22.178ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178342024-02-08T06:46:22.178ZLoading write count information from file "/tmp/test_repair_verify.data"
178352024-02-08T06:46:22.178ZRun Repair workload
178362024-02-08T06:46:22.178Z01/30 Write block 262 len 4096 data: 17
178372024-02-08T06:46:22.182Z02/30 Write block 341 len 16384 data: 17 16 13 12
178382024-02-08T06:46:22.182Z03/30 Read block 499 len 12288
178392024-02-08T06:46:22.182Z04/30 Read block 542 len 20480
178402024-02-08T06:46:22.182Z05/30 Read block 217 len 8192
178412024-02-08T06:46:22.182Z06/30 Read block 74 len 40960
178422024-02-08T06:46:22.182Z07/30 Write block 289 len 40960 data: 11 15 17 14 12 10 8 8 7 9
178432024-02-08T06:46:22.182Z08/30 Read block 532 len 8192
178442024-02-08T06:46:22.182Z09/30 Read block 401 len 36864
178452024-02-08T06:46:22.186Z10/30 Read block 247 len 40960
178462024-02-08T06:46:22.186Z11/30 Read block 590 len 28672
178472024-02-08T06:46:22.186Z12/30 Read block 116 len 12288
178482024-02-08T06:46:22.186Z13/30 Write block 367 len 8192 data: 16 13
178492024-02-08T06:46:22.186Z14/30 Flush
178502024-02-08T06:46:22.189Z15/30 Flush
178512024-02-08T06:46:22.192Z16/30 Write block 416 len 28672 data: 10 13 15 15 14 14 12
178522024-02-08T06:46:22.192Z17/30 Write block 486 len 4096 data: 15
178532024-02-08T06:46:22.192Z18/30 Read block 200 len 28672
178542024-02-08T06:46:22.192Z19/30 Write block 297 len 4096 data: 8
178552024-02-08T06:46:22.192Z20/30 Write block 135 len 16384 data: 8 9 13 13
178562024-02-08T06:46:22.195Z21/30 Write block 388 len 32768 data: 11 14 9 10 12 10 9 8
178572024-02-08T06:46:22.195Z22/30 Flush
178582024-02-08T06:46:22.198Z23/30 Flush
178592024-02-08T06:46:22.198Z24/30 Write block 261 len 36864 data: 18 18 14 14 15 15 17 17 16
178602024-02-08T06:46:22.198Z25/30 Write block 12 len 32768 data: 10 10 9 10 9 9 10 11
178612024-02-08T06:46:22.198Z26/30 Write block 235 len 28672 data: 13 12 11 14 15 13 17
178622024-02-08T06:46:22.202Z27/30 Write block 472 len 24576 data: 15 14 16 15 10 9
178632024-02-08T06:46:22.202Z28/30 Write block 338 len 28672 data: 19 19 17 18 17 14 13
178642024-02-08T06:46:22.202Z29/30 Read block 229 len 24576
178652024-02-08T06:46:22.202Z30/30 Read block 41 len 24576
178662024-02-08T06:46:22.205Z----------------------------------------------------------------
178672024-02-08T06:46:22.205Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:15
178682024-02-08T06:46:22.206ZStates: Active Active Active
178692024-02-08T06:46:22.206ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
178702024-02-08T06:46:22.206Z 16 Acked 1015 Write 7 Done Done Done false
178712024-02-08T06:46:22.206Z 17 Acked 1016 Write 1 Done Done Done false
178722024-02-08T06:46:22.206Z 18 Acked 1017 Read 7 Done Done Done false
178732024-02-08T06:46:22.206Z 19 Acked 1018 Write 1 Sent Done Done false
178742024-02-08T06:46:22.206Z 20 Acked 1019 Write 4 Sent Done Done false
178752024-02-08T06:46:22.206Z 21 Acked 1020 Write 8 Sent Done Done false
178762024-02-08T06:46:22.206Z 22 Acked 1021 Flush 0 Sent Done Done false
178772024-02-08T06:46:22.206Z 23 Acked 1022 Flush 0 Sent Done Done false
178782024-02-08T06:46:22.206Z 24 Acked 1023 Write 9 Sent Done Done false
178792024-02-08T06:46:22.206Z 25 Acked 1024 Write 8 Sent Done Done false
178802024-02-08T06:46:22.206Z 26 Acked 1025 Write 7 Sent Done Done false
178812024-02-08T06:46:22.206Z 27 Acked 1026 Write 6 Sent Done Done false
178822024-02-08T06:46:22.206Z 28 Acked 1027 Write 7 Sent Done Done false
178832024-02-08T06:46:22.207Z 29 Acked 1028 Read 6 Sent Done Done false
178842024-02-08T06:46:22.207Z 30 Acked 1029 Read 6 Sent Sent Done false
178852024-02-08T06:46:22.207Z STATES DS:0 DS:1 DS:2 TOTAL
178862024-02-08T06:46:22.207Z New 0 0 0 0
178872024-02-08T06:46:22.207Z Sent 12 1 0 13
178882024-02-08T06:46:22.207Z Done 3 14 15 32
178892024-02-08T06:46:22.207Z Skipped 0 0 0 0
178902024-02-08T06:46:22.207Z Error 0 0 0 0
178912024-02-08T06:46:22.207ZLast Flush: 1014 1022 1022
178922024-02-08T06:46:22.207ZDownstairs last five completed: 1014 1013 1012 1011 1010
178932024-02-08T06:46:22.207ZUpstairs last five completed: 30 29 28 27 26
178942024-02-08T06:46:22.207Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:21.41817133Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1794}
178952024-02-08T06:46:22.207ZWrote out file "/tmp/test_repair_verify.data"
178962024-02-08T06:46:22.213Z
178972024-02-08T06:46:22.213ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
178982024-02-08T06:46:22.229ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178992024-02-08T06:46:22.229Z 0 000-019 156 156 156 267 267 267 F T T <---
179002024-02-08T06:46:22.229Z 1 020-039 163 163 163 278 278 278 F F F
179012024-02-08T06:46:22.229Z 2 040-059 163 163 163 278 278 278 F F F
179022024-02-08T06:46:22.246Z 3 060-079 157 157 157 268 268 268 F F F
179032024-02-08T06:46:22.246Z 4 080-099 159 159 159 272 272 272 F F F
179042024-02-08T06:46:22.246Z 5 100-119 159 159 159 272 272 272 F F F
179052024-02-08T06:46:22.246Z 6 120-139 160 164 164 273 281 281 F F F <---
179062024-02-08T06:46:22.246Z 7 140-159 162 162 162 277 277 277 F F F
179072024-02-08T06:46:22.247Z 8 160-179 162 162 162 277 277 277 F F F
179082024-02-08T06:46:22.247Z 9 180-199 163 163 163 278 278 278 F F F
179092024-02-08T06:46:22.247Z 10 200-219 162 162 162 277 277 277 F F F
179102024-02-08T06:46:22.247Z 11 220-239 163 163 163 278 278 278 F T T <---
179112024-02-08T06:46:22.247Z 12 240-259 163 163 163 278 278 278 F T T <---
179122024-02-08T06:46:22.247Z 13 260-279 164 164 164 279 279 279 F T T <---
179132024-02-08T06:46:22.247Z 14 280-299 164 164 164 279 281 281 F F F <---
179142024-02-08T06:46:22.247Z 15 300-319 154 154 154 262 262 262 F F F
179152024-02-08T06:46:22.247Z 16 320-339 163 163 163 278 278 278 F T T <---
179162024-02-08T06:46:22.247Z 17 340-359 164 164 164 279 279 279 F T T <---
179172024-02-08T06:46:22.247Z 18 360-379 164 164 164 279 279 279 F F F
179182024-02-08T06:46:22.247Z 19 380-399 162 164 164 276 281 281 F F F <---
179192024-02-08T06:46:22.247Z 20 400-419 153 164 164 260 281 281 T F F <---
179202024-02-08T06:46:22.247Z 21 420-439 161 164 164 275 281 281 T F F <---
179212024-02-08T06:46:22.247Z 22 440-459 163 163 163 278 278 278 F F F
179222024-02-08T06:46:22.248Z 23 460-479 163 163 163 278 278 278 F T T <---
179232024-02-08T06:46:22.248Z 24 480-499 162 164 164 276 281 281 T F F <---
179242024-02-08T06:46:22.248Z 25 500-519 160 160 160 274 274 274 F F F
179252024-02-08T06:46:22.248Z 26 520-539 161 161 161 275 275 275 F F F
179262024-02-08T06:46:22.248Z 27 540-559 163 163 163 278 278 278 F F F
179272024-02-08T06:46:22.248Z 28 560-579 160 160 160 273 273 273 F F F
179282024-02-08T06:46:22.248Z 29 580-599 160 160 160 273 273 273 F F F
179292024-02-08T06:46:22.248ZMax gen: 164, Max flush: 281
179302024-02-08T06:46:22.248ZError: Difference in extent metadata found!
179312024-02-08T06:46:22.248ZA Difference in extent metadata is expected here
179322024-02-08T06:46:22.248ZOn loop 82
179332024-02-08T06:46:22.248Z
179342024-02-08T06:46:22.248ZVerifying data now
179352024-02-08T06:46:27.880ZNo change after verify
179362024-02-08T06:46:27.880ZLoop: 82 Downstairs dump after verify (and repair):
179372024-02-08T06:46:27.894ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179382024-02-08T06:46:27.894Z 0 000-019 165 165 165 282 282 282 F F F
179392024-02-08T06:46:27.895Z 1 020-039 163 163 163 278 278 278 F F F
179402024-02-08T06:46:27.895Z 2 040-059 163 163 163 278 278 278 F F F
179412024-02-08T06:46:27.895Z 3 060-079 157 157 157 268 268 268 F F F
179422024-02-08T06:46:27.895Z 4 080-099 159 159 159 272 272 272 F F F
179432024-02-08T06:46:27.895Z 5 100-119 159 159 159 272 272 272 F F F
179442024-02-08T06:46:27.895Z 6 120-139 164 164 164 281 281 281 F F F
179452024-02-08T06:46:27.895Z 7 140-159 162 162 162 277 277 277 F F F
179462024-02-08T06:46:27.898Z 8 160-179 162 162 162 277 277 277 F F F
179472024-02-08T06:46:27.898Z 9 180-199 163 163 163 278 278 278 F F F
179482024-02-08T06:46:27.898Z 10 200-219 162 162 162 277 277 277 F F F
179492024-02-08T06:46:27.898Z 11 220-239 165 165 165 282 282 282 F F F
179502024-02-08T06:46:27.898Z 12 240-259 165 165 165 282 282 282 F F F
179512024-02-08T06:46:27.898Z 13 260-279 165 165 165 282 282 282 F F F
179522024-02-08T06:46:27.898Z 14 280-299 164 164 164 281 281 281 F F F
179532024-02-08T06:46:27.898Z 15 300-319 154 154 154 262 262 262 F F F
179542024-02-08T06:46:27.899Z 16 320-339 165 165 165 282 282 282 F F F
179552024-02-08T06:46:27.899Z 17 340-359 165 165 165 282 282 282 F F F
179562024-02-08T06:46:27.899Z 18 360-379 164 164 164 279 279 279 F F F
179572024-02-08T06:46:27.899Z 19 380-399 164 164 164 281 281 281 F F F
179582024-02-08T06:46:27.899Z 20 400-419 164 164 164 281 281 281 F F F
179592024-02-08T06:46:27.899Z 21 420-439 164 164 164 281 281 281 F F F
179602024-02-08T06:46:27.899Z 22 440-459 163 163 163 278 278 278 F F F
179612024-02-08T06:46:27.899Z 23 460-479 165 165 165 282 282 282 F F F
179622024-02-08T06:46:27.899Z 24 480-499 164 164 164 281 281 281 F F F
179632024-02-08T06:46:27.899Z 25 500-519 160 160 160 274 274 274 F F F
179642024-02-08T06:46:27.899Z 26 520-539 161 161 161 275 275 275 F F F
179652024-02-08T06:46:27.899Z 27 540-559 163 163 163 278 278 278 F F F
179662024-02-08T06:46:27.899Z 28 560-579 160 160 160 273 273 273 F F F
179672024-02-08T06:46:27.899Z 29 580-599 160 160 160 273 273 273 F F F
179682024-02-08T06:46:27.899ZMax gen: 165, Max flush: 282
179692024-02-08T06:46:27.900Z
179702024-02-08T06:46:27.900ZBegin loop 83
179712024-02-08T06:46:27.900ZDownstairs to restart: 2
179722024-02-08T06:46:27.916Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.130822286Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802}
179732024-02-08T06:46:27.916Z{"msg":"Crucible Version: BuildInfo {\nCrucible runtime is spawned
179742024-02-08T06:46:27.923Z version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131147621Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802}
179752024-02-08T06:46:27.923Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131196068Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802}
179762024-02-08T06:46:27.924Z{"msg":"Crucible 13f50704-ebf9-4c14-8f50-b9f2c902cf0e has session id: 35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131227895Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179772024-02-08T06:46:27.924Z{"msg":"Upstairs opts: Upstairs UUID: 13f50704-ebf9-4c14-8f50-b9f2c902cf0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131283001Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179782024-02-08T06:46:27.924Z{"msg":"Crucible stats registered with UUID: 13f50704-ebf9-4c14-8f50-b9f2c902cf0e","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131328828Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179792024-02-08T06:46:27.924Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131378964Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802}
179802024-02-08T06:46:27.924Z{"msg":"13f50704-ebf9-4c14-8f50-b9f2c902cf0e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131422401Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179812024-02-08T06:46:27.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131462828Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179822024-02-08T06:46:27.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131520863Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179832024-02-08T06:46:27.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131573789Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179842024-02-08T06:46:27.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131620336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179852024-02-08T06:46:27.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131655863Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179862024-02-08T06:46:27.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131687041Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179872024-02-08T06:46:27.924Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131715509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179882024-02-08T06:46:27.924Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131741887Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179892024-02-08T06:46:27.924Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131778104Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179902024-02-08T06:46:27.924Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131795263Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179912024-02-08T06:46:27.925Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.13182981Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179922024-02-08T06:46:27.925Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.131862708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179932024-02-08T06:46:27.925Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.132325413Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179942024-02-08T06:46:27.925Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.13236162Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179952024-02-08T06:46:27.925Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.132377369Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179962024-02-08T06:46:27.925Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.132399197Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179972024-02-08T06:46:27.925Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.132925338Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179982024-02-08T06:46:27.925Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.133072517Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
179992024-02-08T06:46:27.925Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.133487336Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180002024-02-08T06:46:27.925Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.133540382Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180012024-02-08T06:46:27.925Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.133585638Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180022024-02-08T06:46:27.925Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.133628125Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180032024-02-08T06:46:27.925Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.133672752Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180042024-02-08T06:46:27.925Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.133713799Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180052024-02-08T06:46:27.925Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.133746286Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180062024-02-08T06:46:27.925Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.133764085Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180072024-02-08T06:46:27.926Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134214511Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180082024-02-08T06:46:27.930Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134651978Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180092024-02-08T06:46:27.930Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134688545Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180102024-02-08T06:46:27.930Z{"msg":"[0]R flush_numbers[0..12]: [282, 278, 278, 268, 272, 272, 281, 277, 277, 278, 277, 282]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134709574Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180112024-02-08T06:46:27.930Z{"msg":"[0]R generation[0..12]: [165, 163, 163, 157, 159, 159, 164, 162, 162, 163, 162, 165]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134742192Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180122024-02-08T06:46:27.931Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.13476453Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180132024-02-08T06:46:27.931Z{"msg":"[1]R flush_numbers[0..12]: [282, 278, 278, 268, 272, 272, 281, 277, 277, 278, 277, 282]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134793837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180142024-02-08T06:46:27.931Z{"msg":"[1]R generation[0..12]: [165, 163, 163, 157, 159, 159, 164, 162, 162, 163, 162, 165]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
180152024-02-08T06:46:27.931Z,"time":"2024-02-08T06:46:27.134814476Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180162024-02-08T06:46:27.931Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134841464Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180172024-02-08T06:46:27.931Z{"msg":"[2]R flush_numbers[0..12]: [282, 278, 278, 268, 272, 272, 281, 277, 277, 278, 277, 282]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134872912Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180182024-02-08T06:46:27.931Z{"msg":"[2]R generation[0..12]: [165, 163, 163, 157, 159, 159, 164, 162, 162, 163, 162, 165]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134903879Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180192024-02-08T06:46:27.931Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134921248Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180202024-02-08T06:46:27.931Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134940997Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180212024-02-08T06:46:27.931Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134969714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180222024-02-08T06:46:27.931Z{"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.134997442Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180232024-02-08T06:46:27.931Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.135021291Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180242024-02-08T06:46:27.931Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.135036719Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180252024-02-08T06:46:27.931Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.135065337Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180262024-02-08T06:46:27.932Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.135097865Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180272024-02-08T06:46:27.932Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.135129052Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180282024-02-08T06:46:27.932Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.13516096Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180292024-02-08T06:46:27.932Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.135190658Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180302024-02-08T06:46:27.932ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
180312024-02-08T06:46:27.932Z{"msg":"13f50704-ebf9-4c14-8f50-b9f2c902cf0e is now active with session: 35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.135224505Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180322024-02-08T06:46:27.932Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.135255323Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802,"session_id":"35ee557d-c3d4-4a64-90f7-c8e1ba3a59b2"}
180332024-02-08T06:46:27.932Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:27.135283221Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802}
180342024-02-08T06:46:27.932ZLoading write count information from file "/tmp/test_repair_verify.data"
180352024-02-08T06:46:27.932ZRun Repair workload
180362024-02-08T06:46:27.932Z01/30 Write block 125 len 24576 data: 11 8 7 9 10 11
180372024-02-08T06:46:27.932Z02/30 Read block 171 len 40960
180382024-02-08T06:46:27.932Z03/30 Write block 591 len 28672 data: 14 12 13 12 9 9 8
180392024-02-08T06:46:27.932Z04/30 Read block 241 len 32768
180402024-02-08T06:46:27.932Z05/30 Flush
180412024-02-08T06:46:27.936Z06/30 Write block 75 len 12288 data: 12 12 9
180422024-02-08T06:46:27.936Z07/30 Write block 58 len 24576 data: 10 9 9 9 10 10
180432024-02-08T06:46:27.936Z08/30 Read block 239 len 4096
180442024-02-08T06:46:27.936Z09/30 Read block 273 len 40960
180452024-02-08T06:46:27.936Z10/30 Read block 572 len 12288
180462024-02-08T06:46:27.936Z11/30 Read block 485 len 24576
180472024-02-08T06:46:27.936Z12/30 Read block 175 len 32768
180482024-02-08T06:46:27.936Z13/30 Read block 208 len 40960
180492024-02-08T06:46:27.939Z14/30 Write block 261 len 24576 data: 19 19 15 15 16 16
180502024-02-08T06:46:27.939Z15/30 Read block 402 len 20480
180512024-02-08T06:46:27.939Z16/30 Write block 113 len 32768 data: 14 13 14 15 12 11 11 10
180522024-02-08T06:46:27.940Z17/30 Read block 304 len 24576
180532024-02-08T06:46:27.942Z18/30 Read block 495 len 24576
180542024-02-08T06:46:27.943Z19/30 Write block 371 len 32768 data: 14 18 16 8 11 12 10 10
180552024-02-08T06:46:27.946Z20/30 Write block 448 len 32768 data: 13 12 10 11 12 14 14 18
180562024-02-08T06:46:27.946Z21/30 Read block 357 len 4096
180572024-02-08T06:46:27.946Z22/30 Write block 343 len 28672 data: 15 14 14 15 12 11 12
180582024-02-08T06:46:27.946Z23/30 Read block 579 len 24576
180592024-02-08T06:46:27.949Z24/30 Read block 11 len 32768
180602024-02-08T06:46:27.949Z25/30 Read block 240 len 24576
180612024-02-08T06:46:27.949Z26/30 Read block 538 len 8192
180622024-02-08T06:46:27.949Z27/30 Read block 575 len 36864
180632024-02-08T06:46:27.953Z28/30 Read block 308 len 4096
180642024-02-08T06:46:27.953Z29/30 Read block 78 len 16384
180652024-02-08T06:46:27.953Z30/30 Read block 295 len 40960
180662024-02-08T06:46:27.953Z----------------------------------------------------------------
180672024-02-08T06:46:27.953Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:25
180682024-02-08T06:46:27.953ZStates: Active Active Active
180692024-02-08T06:46:27.953ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
180702024-02-08T06:46:27.953Z 6 Acked 1005 Write 3 Done Done Sent false
180712024-02-08T06:46:27.953Z 7 Acked 1006 Write 6 Done Done Sent false
180722024-02-08T06:46:27.953Z 8 Acked 1007 Read 1 Done Done Sent false
180732024-02-08T06:46:27.953Z 9 Acked 1008 Read 10 Done Done Sent false
180742024-02-08T06:46:27.953Z 10 Acked 1009 Read 3 Done Done Sent false
180752024-02-08T06:46:27.953Z 11 Acked 1010 Read 6 Done Done Sent false
180762024-02-08T06:46:27.953Z 12 Acked 1011 Read 8 Done Done Sent false
180772024-02-08T06:46:27.954Z 13 Acked 1012 Read 10 Done Done Sent false
180782024-02-08T06:46:27.954Z 14 Acked 1013 Write 6 Done Done Sent false
180792024-02-08T06:46:27.954Z 15 Acked 1014 Read 5 Done Done Sent false
180802024-02-08T06:46:27.954Z 16 Acked 1015 Write 8 Done Done Sent false
180812024-02-08T06:46:27.954Z 17 Acked 1016 Read 6 Done Done Sent false
180822024-02-08T06:46:27.954Z 18 Acked 1017 Read 6 Done Done Sent false
180832024-02-08T06:46:27.954Z 19 Acked 1018 Write 8 Done Done Sent false
180842024-02-08T06:46:27.954Z 20 Acked 1019 Write 8 Done Done Sent false
180852024-02-08T06:46:27.954Z 21 Acked 1020 Read 1 Done Done Sent false
180862024-02-08T06:46:27.954Z 22 Acked 1021 Write 7 Done Done Sent false
180872024-02-08T06:46:27.954Z 23 Acked 1022 Read 6 Done Done Sent false
180882024-02-08T06:46:27.954Z 24 Acked 1023 Read 8 Done Done Sent false
180892024-02-08T06:46:27.954Z 25 Acked 1024 Read 6 Done Done Sent false
180902024-02-08T06:46:27.954Z 26 Acked 1025 Read 2 Done Done Sent false
180912024-02-08T06:46:27.954Z 27 Acked 1026 Read 9 Done Done Sent false
180922024-02-08T06:46:27.955Z 28 Acked 1027 Read 1 Done Done Sent false
180932024-02-08T06:46:27.955Z 29 Acked 1028 Read 4 Done Done Sent false
180942024-02-08T06:46:27.955Z 30 Acked 1029 Read 10 Done Sent Sent false
180952024-02-08T06:46:27.955Z STATES DS:0 DS:1 DS:2 TOTAL
180962024-02-08T06:46:27.955Z New 0 0 0 0
180972024-02-08T06:46:27.955Z Sent 0 1 25 26
180982024-02-08T06:46:27.955Z Done 25 24 0 49
180992024-02-08T06:46:27.955Z Skipped 0 0 0 0
181002024-02-08T06:46:27.955Z Error 0 0 0 0
181012024-02-08T06:46:27.955ZLast Flush: 1004 1004 1004
181022024-02-08T06:46:27.955ZDownstairs last five completed: 1004 1003 1002 1001 1000
181032024-02-08T06:46:27.955ZUpstairs last five completed: 30 29 28 27 26
181042024-02-08T06:46:27.955Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:27.166892478Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1802}
181052024-02-08T06:46:27.955ZWrote out file "/tmp/test_repair_verify.data"
181062024-02-08T06:46:27.958Z
181072024-02-08T06:46:27.961ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181082024-02-08T06:46:27.977ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181092024-02-08T06:46:27.977Z 0 000-019 165 165 165 282 282 282 F F F
181102024-02-08T06:46:27.977Z 1 020-039 163 163 163 278 278 278 F F F
181112024-02-08T06:46:27.977Z 2 040-059 163 163 163 278 278 278 T T F <---
181122024-02-08T06:46:27.977Z 3 060-079 157 157 157 268 268 268 T T F <---
181132024-02-08T06:46:27.977Z 4 080-099 159 159 159 272 272 272 F F F
181142024-02-08T06:46:27.977Z 5 100-119 159 159 159 272 272 272 T T F <---
181152024-02-08T06:46:27.977Z 6 120-139 166 166 166 283 283 283 T T F <---
181162024-02-08T06:46:27.981Z 7 140-159 162 162 162 277 277 277 F F F
181172024-02-08T06:46:27.981Z 8 160-179 162 162 162 277 277 277 F F F
181182024-02-08T06:46:27.981Z 9 180-199 163 163 163 278 278 278 F F F
181192024-02-08T06:46:27.981Z 10 200-219 162 162 162 277 277 277 F F F
181202024-02-08T06:46:27.981Z 11 220-239 165 165 165 282 282 282 F F F
181212024-02-08T06:46:27.981Z 12 240-259 165 165 165 282 282 282 F F F
181222024-02-08T06:46:27.981Z 13 260-279 165 165 165 282 282 282 T T F <---
181232024-02-08T06:46:27.981Z 14 280-299 164 164 164 281 281 281 F F F
181242024-02-08T06:46:27.981Z 15 300-319 154 154 154 262 262 262 F F F
181252024-02-08T06:46:27.981Z 16 320-339 165 165 165 282 282 282 F F F
181262024-02-08T06:46:27.981Z 17 340-359 165 165 165 282 282 282 T T F <---
181272024-02-08T06:46:27.981Z 18 360-379 164 164 164 279 279 279 T T F <---
181282024-02-08T06:46:27.982Z 19 380-399 164 164 164 281 281 281 F F F
181292024-02-08T06:46:27.982Z 20 400-419 164 164 164 281 281 281 F F F
181302024-02-08T06:46:27.982Z 21 420-439 164 164 164 281 281 281 F F F
181312024-02-08T06:46:27.982Z 22 440-459 163 163 163 278 278 278 T T F <---
181322024-02-08T06:46:27.982Z 23 460-479 165 165 165 282 282 282 F F F
181332024-02-08T06:46:27.982Z 24 480-499 164 164 164 281 281 281 F F F
181342024-02-08T06:46:27.982Z 25 500-519 160 160 160 274 274 274 F F F
181352024-02-08T06:46:27.982Z 26 520-539 161 161 161 275 275 275 F F F
181362024-02-08T06:46:27.982Z 27 540-559 163 163 163 278 278 278 F F F
181372024-02-08T06:46:27.982Z 28 560-579 160 160 160 273 273 273 F F F
181382024-02-08T06:46:27.982Z 29 580-599 166 166 166 283 283 283 F F F
181392024-02-08T06:46:27.982ZMax gen: 166, Max flush: 283
181402024-02-08T06:46:27.982ZError: Difference in extent metadata found!
181412024-02-08T06:46:27.982ZA Difference in extent metadata is expected here
181422024-02-08T06:46:27.983ZOn loop 83
181432024-02-08T06:46:27.983Z
181442024-02-08T06:46:27.985ZVerifying data now
181452024-02-08T06:46:33.442ZNo change after verify
181462024-02-08T06:46:33.442ZLoop: 83 Downstairs dump after verify (and repair):
181472024-02-08T06:46:33.457ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181482024-02-08T06:46:33.457Z 0 000-019 165 165 165 282 282 282 F F F
181492024-02-08T06:46:33.457Z 1 020-039 163 163 163 278 278 278 F F F
181502024-02-08T06:46:33.457Z 2 040-059 167 167 167 284 284 284 F F F
181512024-02-08T06:46:33.457Z 3 060-079 167 167 167 284 284 284 F F F
181522024-02-08T06:46:33.458Z 4 080-099 159 159 159 272 272 272 F F F
181532024-02-08T06:46:33.458Z 5 100-119 167 167 167 284 284 284 F F F
181542024-02-08T06:46:33.461Z 6 120-139 167 167 167 284 284 284 F F F
181552024-02-08T06:46:33.461Z 7 140-159 162 162 162 277 277 277 F F F
181562024-02-08T06:46:33.461Z 8 160-179 162 162 162 277 277 277 F F F
181572024-02-08T06:46:33.461Z 9 180-199 163 163 163 278 278 278 F F F
181582024-02-08T06:46:33.461Z 10 200-219 162 162 162 277 277 277 F F F
181592024-02-08T06:46:33.461Z 11 220-239 165 165 165 282 282 282 F F F
181602024-02-08T06:46:33.461Z 12 240-259 165 165 165 282 282 282 F F F
181612024-02-08T06:46:33.461Z 13 260-279 167 167 167 284 284 284 F F F
181622024-02-08T06:46:33.461Z 14 280-299 164 164 164 281 281 281 F F F
181632024-02-08T06:46:33.462Z 15 300-319 154 154 154 262 262 262 F F F
181642024-02-08T06:46:33.462Z 16 320-339 165 165 165 282 282 282 F F F
181652024-02-08T06:46:33.462Z 17 340-359 167 167 167 284 284 284 F F F
181662024-02-08T06:46:33.462Z 18 360-379 167 167 167 284 284 284 F F F
181672024-02-08T06:46:33.462Z 19 380-399 164 164 164 281 281 281 F F F
181682024-02-08T06:46:33.462Z 20 400-419 164 164 164 281 281 281 F F F
181692024-02-08T06:46:33.462Z 21 420-439 164 164 164 281 281 281 F F F
181702024-02-08T06:46:33.462Z 22 440-459 167 167 167 284 284 284 F F F
181712024-02-08T06:46:33.462Z 23 460-479 165 165 165 282 282 282 F F F
181722024-02-08T06:46:33.462Z 24 480-499 164 164 164 281 281 281 F F F
181732024-02-08T06:46:33.462Z 25 500-519 160 160 160 274 274 274 F F F
181742024-02-08T06:46:33.462Z 26 520-539 161 161 161 275 275 275 F F F
181752024-02-08T06:46:33.462Z 27 540-559 163 163 163 278 278 278 F F F
181762024-02-08T06:46:33.462Z 28 560-579 160 160 160 273 273 273 F F F
181772024-02-08T06:46:33.462Z 29 580-599 166 166 166 283 283 283 F F F
181782024-02-08T06:46:33.463ZMax gen: 167, Max flush: 284
181792024-02-08T06:46:33.463Z
181802024-02-08T06:46:33.463ZBegin loop 84
181812024-02-08T06:46:33.463ZDownstairs to restart: 2
181822024-02-08T06:46:33.479Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693157567Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810}
181832024-02-08T06:46:33.479ZCrucible runtime is spawned
181842024-02-08T06:46:33.484Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693502102Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810}
181852024-02-08T06:46:33.484Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693547148Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810}
181862024-02-08T06:46:33.484Z{"msg":"Crucible 094c195e-0aa1-41e2-a246-9374b19a261a has session id: c13e9d78-793a-4139-a0ea-abc322c07949","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693571556Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181872024-02-08T06:46:33.484Z{"msg":"Upstairs opts: Upstairs UUID: 094c195e-0aa1-41e2-a246-9374b19a261a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693609714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181882024-02-08T06:46:33.484Z{"msg":"Crucible stats registered with UUID: 094c195e-0aa1-41e2-a246-9374b19a261a","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693628082Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181892024-02-08T06:46:33.484Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.69365283Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810}
181902024-02-08T06:46:33.484Z{"msg":"094c195e-0aa1-41e2-a246-9374b19a261a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693672669Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181912024-02-08T06:46:33.484Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693706046Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181922024-02-08T06:46:33.484Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693744014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181932024-02-08T06:46:33.484Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693778411Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181942024-02-08T06:46:33.484Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693825647Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181952024-02-08T06:46:33.484Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693869114Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181962024-02-08T06:46:33.485Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693913121Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181972024-02-08T06:46:33.485Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693946838Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181982024-02-08T06:46:33.485Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.693999454Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
181992024-02-08T06:46:33.485Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.694174221Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182002024-02-08T06:46:33.485Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:32.694238916Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182012024-02-08T06:46:33.485Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:32.694274504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182022024-02-08T06:46:33.485Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:32.694306981Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182032024-02-08T06:46:33.485Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.6943274Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182042024-02-08T06:46:33.485Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.694352098Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182052024-02-08T06:46:33.485Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.694375056Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182062024-02-08T06:46:33.485Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.694415593Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182072024-02-08T06:46:33.485Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.694502187Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182082024-02-08T06:46:33.485Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.69472424Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182092024-02-08T06:46:33.485Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.694758557Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182102024-02-08T06:46:33.485Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.694775816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182112024-02-08T06:46:33.486Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.694847771Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182122024-02-08T06:46:33.486Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.695281428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182132024-02-08T06:46:33.486Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.69538804Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182142024-02-08T06:46:33.486Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.695777641Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182152024-02-08T06:46:33.486Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.695816908Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182162024-02-08T06:46:33.486Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.695832887Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182172024-02-08T06:46:33.486Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.695857365Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182182024-02-08T06:46:33.486Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.695880803Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182192024-02-08T06:46:33.486Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:32.695895402Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182202024-02-08T06:46:34.482Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.69555333Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182212024-02-08T06:46:34.487Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.695866286Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182222024-02-08T06:46:34.487Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.696810815Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182232024-02-08T06:46:34.487Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.696845743Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182242024-02-08T06:46:34.487Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.697329606Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182252024-02-08T06:46:34.487Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.697874575Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182262024-02-08T06:46:34.487Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.697919422Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182272024-02-08T06:46:34.487Z{"msg":"[0]R flush_numbers[0..12]: [282, 278, 284, 284, 272, 284, 284, 277, 277, 278, 277, 282]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.697937691Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182282024-02-08T06:46:34.487Z{"msg":"[0]R generation[0..12]: [165, 163, 167, 167, 159, 167, 167, 162, 162, 163, 162, 165]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.697960169Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182292024-02-08T06:46:34.487Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.697975668Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182302024-02-08T06:46:34.487Z{"msg":"[1]R flush_numbers[0..12]: [282, 278, 284, 284, 272, 284, 284, 277, 277, 278, 277, 282]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.697990457Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182312024-02-08T06:46:34.488Z{"msg":"[1]R generation[0..12]: [165, 163, 167, 167, 159, 167, 167, 162, 162, 163, 162, 165]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698004746Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182322024-02-08T06:46:34.488Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698019135Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182332024-02-08T06:46:34.488Z{"msg":"[2]R flush_numbers[0..12]: [282, 278, 284, 284, 272, 284, 284, 277, 277, 278, 277, 282]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698040833Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182342024-02-08T06:46:34.488Z{"msg":"[2]R generation[0..12]: [165, 163, 167, 167, 159, 167, 167, 162, 162, 163, 162, 165]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698055122Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182352024-02-08T06:46:34.488Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698069521Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182362024-02-08T06:46:34.488Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698089699Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182372024-02-08T06:46:34.488Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
182382024-02-08T06:46:34.488Ztime":"2024-02-08T06:46:33.698104358Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182392024-02-08T06:46:34.488Z{"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698120997Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182402024-02-08T06:46:34.488Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698135096Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182412024-02-08T06:46:34.488Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698148735Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182422024-02-08T06:46:34.488Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698161354Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182432024-02-08T06:46:34.488Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698182452Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182442024-02-08T06:46:34.488Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698196401Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182452024-02-08T06:46:34.488Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.6982106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182462024-02-08T06:46:34.489Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698230869Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182472024-02-08T06:46:34.489Z{"msg":"094c195e-0aa1-41e2-a246-9374b19a261a is now active with session: c13e9d78-793a-4139-a0ea-abc322c07949","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698244708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182482024-02-08T06:46:34.489Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698264036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810,"session_id":"c13e9d78-793a-4139-a0ea-abc322c07949"}
182492024-02-08T06:46:34.489Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:33.698276775Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810}
182502024-02-08T06:46:34.489ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
182512024-02-08T06:46:34.489ZLoading write count information from file "/tmp/test_repair_verify.data"
182522024-02-08T06:46:34.489ZRun Repair workload
182532024-02-08T06:46:34.489Z01/30 Flush
182542024-02-08T06:46:34.492Z02/30 Write block 527 len 36864 data: 12 13 13 12 13 15 14 14 15
182552024-02-08T06:46:34.492Z03/30 Write block 114 len 12288 data: 14 15 16
182562024-02-08T06:46:34.492Z04/30 Write block 235 len 20480 data: 14 13 12 15 16
182572024-02-08T06:46:34.492Z05/30 Read block 513 len 20480
182582024-02-08T06:46:34.493Z06/30 Write block 341 len 20480 data: 19 18 16 15 15
182592024-02-08T06:46:34.493Z07/30 Read block 86 len 32768
182602024-02-08T06:46:34.493Z08/30 Read block 544 len 40960
182612024-02-08T06:46:34.493Z09/30 Flush
182622024-02-08T06:46:34.496Z10/30 Write block 30 len 24576 data: 16 16 14 14 12 14
182632024-02-08T06:46:34.496Z11/30 Read block 161 len 28672
182642024-02-08T06:46:34.498Z12/30 Write block 10 len 12288 data: 13 9 11
182652024-02-08T06:46:34.499Z13/30 Read block 427 len 24576
182662024-02-08T06:46:34.499Z14/30 Flush
182672024-02-08T06:46:34.501Z15/30 Write block 35 len 16384 data: 15 13 11 13
182682024-02-08T06:46:34.502Z16/30 Write block 272 len 20480 data: 10 12 11 13 15
182692024-02-08T06:46:34.502Z17/30 Read block 168 len 32768
182702024-02-08T06:46:34.504Z18/30 Write block 428 len 32768 data: 7 7 11 10 11 8 11 12
182712024-02-08T06:46:34.504Z19/30 Write block 373 len 36864 data: 17 9 12 13 11 11 10 10 7
182722024-02-08T06:46:34.504Z20/30 Write block 472 len 28672 data: 16 15 17 16 11 10 8
182732024-02-08T06:46:34.507Z21/30 Read block 328 len 24576
182742024-02-08T06:46:34.507Z22/30 Read block 31 len 40960
182752024-02-08T06:46:34.507Z23/30 Read block 233 len 4096
182762024-02-08T06:46:34.510Z24/30 Write block 551 len 40960 data: 25 23 21 18 18 21 14 10 12 14
182772024-02-08T06:46:34.510Z25/30 Write block 11 len 16384 data: 10 12 11 10
182782024-02-08T06:46:34.510Z26/30 Read block 117 len 12288
182792024-02-08T06:46:34.510Z27/30 Write block 314 len 8192 data: 10 6
182802024-02-08T06:46:34.514Z28/30 Read block 304 len 28672
182812024-02-08T06:46:34.514Z29/30 Write block 393 len 28672 data: 11 10 9 6 9 11 12
182822024-02-08T06:46:34.514Z30/30 Write block 90 len 8192 data: 11 14
182832024-02-08T06:46:34.514Z----------------------------------------------------------------
182842024-02-08T06:46:34.514Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:29
182852024-02-08T06:46:34.514ZStates: Active Active Active
182862024-02-08T06:46:34.514ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182872024-02-08T06:46:34.514Z 2 Acked 1001 Write 9 Done Done Done false
182882024-02-08T06:46:34.514Z 3 Acked 1002 Write 3 Done Done Done false
182892024-02-08T06:46:34.514Z 4 Acked 1003 Write 5 Done Done Done false
182902024-02-08T06:46:34.514Z 5 Acked 1004 Read 5 Done Done Sent false
182912024-02-08T06:46:34.514Z 6 Acked 1005 Write 5 Done Done Sent false
182922024-02-08T06:46:34.515Z 7 Acked 1006 Read 8 Done Done Sent false
182932024-02-08T06:46:34.515Z 8 Acked 1007 Read 10 Done Done Sent false
182942024-02-08T06:46:34.515Z 9 Acked 1008 Flush 0 Done Done Sent false
182952024-02-08T06:46:34.515Z 10 Acked 1009 Write 6 Done Done Sent false
182962024-02-08T06:46:34.515Z 11 Acked 1010 Read 7 Done Done Sent false
182972024-02-08T06:46:34.515Z 12 Acked 1011 Write 3 Done Done Sent false
182982024-02-08T06:46:34.515Z 13 Acked 1012 Read 6 Done Done Sent false
182992024-02-08T06:46:34.515Z 14 Acked 1013 Flush 0 Done Done Sent false
183002024-02-08T06:46:34.515Z 15 Acked 1014 Write 4 Done Done Sent false
183012024-02-08T06:46:34.515Z 16 Acked 1015 Write 5 Done Done Sent false
183022024-02-08T06:46:34.515Z 17 Acked 1016 Read 8 Done Done Sent false
183032024-02-08T06:46:34.515Z 18 Acked 1017 Write 8 Done Done Sent false
183042024-02-08T06:46:34.515Z 19 Acked 1018 Write 9 Done Done Sent false
183052024-02-08T06:46:34.515Z 20 Acked 1019 Write 7 Done Done Sent false
183062024-02-08T06:46:34.515Z 21 Acked 1020 Read 6 Done Done Sent false
183072024-02-08T06:46:34.515Z 22 Acked 1021 Read 10 Done Done Sent false
183082024-02-08T06:46:34.516Z 23 Acked 1022 Read 1 Done Done Sent false
183092024-02-08T06:46:34.516Z 24 Acked 1023 Write 10 Done Done Sent false
183102024-02-08T06:46:34.516Z 25 Acked 1024 Write 4 Done Done Sent false
183112024-02-08T06:46:34.516Z 26 Acked 1025 Read 3 Done Done Sent false
183122024-02-08T06:46:34.516Z 27 Acked 1026 Write 2 Done Done Sent false
183132024-02-08T06:46:34.516Z 28 Acked 1027 Read 7 Done Done Sent false
183142024-02-08T06:46:34.516Z 29 Acked 1028 Write 7 Sent Sent Sent false
183152024-02-08T06:46:34.516Z 30 Acked 1029 Write 2 Sent Sent Sent false
183162024-02-08T06:46:34.516Z STATES DS:0 DS:1 DS:2 TOTAL
183172024-02-08T06:46:34.516Z New 0 0 0 0
183182024-02-08T06:46:34.516Z Sent 2 2 26 30
183192024-02-08T06:46:34.516Z Done 27 27 3 57
183202024-02-08T06:46:34.516Z Skipped 0 0 0 0
183212024-02-08T06:46:34.516Z Error 0 0 0 0
183222024-02-08T06:46:34.516ZLast Flush: 1013 1013 1000
183232024-02-08T06:46:34.517ZDownstairs last five completed: 1000
183242024-02-08T06:46:34.517ZUpstairs last five completed: 30 29 28 27 26
183252024-02-08T06:46:34.517Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:33.727496891Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1810}
183262024-02-08T06:46:34.520ZWrote out file "/tmp/test_repair_verify.data"
183272024-02-08T06:46:34.520Z
183282024-02-08T06:46:34.523ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
183292024-02-08T06:46:34.542ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183302024-02-08T06:46:34.542Z 0 000-019 168 168 165 287 287 282 T T F <---
183312024-02-08T06:46:34.542Z 1 020-039 168 168 163 287 287 278 T T F <---
183322024-02-08T06:46:34.542Z 2 040-059 167 167 167 284 284 284 F F F
183332024-02-08T06:46:34.542Z 3 060-079 167 167 167 284 284 284 F F F
183342024-02-08T06:46:34.542Z 4 080-099 159 159 159 272 272 272 T T F <---
183352024-02-08T06:46:34.542Z 5 100-119 168 168 167 286 286 284 F F T <---
183362024-02-08T06:46:34.542Z 6 120-139 167 167 167 284 284 284 F F F
183372024-02-08T06:46:34.545Z 7 140-159 162 162 162 277 277 277 F F F
183382024-02-08T06:46:34.545Z 8 160-179 162 162 162 277 277 277 F F F
183392024-02-08T06:46:34.546Z 9 180-199 163 163 163 278 278 278 F F F
183402024-02-08T06:46:34.546Z 10 200-219 162 162 162 277 277 277 F F F
183412024-02-08T06:46:34.546Z 11 220-239 168 168 165 286 286 282 F F T <---
183422024-02-08T06:46:34.546Z 12 240-259 165 165 165 282 282 282 F F F
183432024-02-08T06:46:34.546Z 13 260-279 167 167 167 284 284 284 T T F <---
183442024-02-08T06:46:34.546Z 14 280-299 164 164 164 281 281 281 F F F
183452024-02-08T06:46:34.546Z 15 300-319 154 154 154 262 262 262 T T F <---
183462024-02-08T06:46:34.546Z 16 320-339 165 165 165 282 282 282 F F F
183472024-02-08T06:46:34.546Z 17 340-359 168 168 167 286 286 284 F F F <---
183482024-02-08T06:46:34.546Z 18 360-379 167 167 167 284 284 284 T T F <---
183492024-02-08T06:46:34.546Z 19 380-399 164 164 164 281 281 281 T T F <---
183502024-02-08T06:46:34.546Z 20 400-419 164 164 164 281 281 281 F F F
183512024-02-08T06:46:34.546Z 21 420-439 164 164 164 281 281 281 T T F <---
183522024-02-08T06:46:34.546Z 22 440-459 167 167 167 284 284 284 F F F
183532024-02-08T06:46:34.547Z 23 460-479 165 165 165 282 282 282 T T F <---
183542024-02-08T06:46:34.547Z 24 480-499 164 164 164 281 281 281 F F F
183552024-02-08T06:46:34.547Z 25 500-519 160 160 160 274 274 274 F F F
183562024-02-08T06:46:34.547Z 26 520-539 168 168 161 286 286 275 F F T <---
183572024-02-08T06:46:34.547Z 27 540-559 163 163 163 278 278 278 T T F <---
183582024-02-08T06:46:34.547Z 28 560-579 160 160 160 273 273 273 T T F <---
183592024-02-08T06:46:34.547Z 29 580-599 166 166 166 283 283 283 F F F
183602024-02-08T06:46:34.547ZMax gen: 168, Max flush: 287
183612024-02-08T06:46:34.547ZError: Difference in extent metadata found!
183622024-02-08T06:46:34.547ZA Difference in extent metadata is expected here
183632024-02-08T06:46:34.547ZOn loop 84
183642024-02-08T06:46:34.547Z
183652024-02-08T06:46:34.550ZVerifying data now
183662024-02-08T06:46:40.289ZNo change after verify
183672024-02-08T06:46:40.289ZLoop: 84 Downstairs dump after verify (and repair):
183682024-02-08T06:46:40.717ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183692024-02-08T06:46:40.717Z 0 000-019 169 169 169 288 288 288 F F F
183702024-02-08T06:46:40.717Z 1 020-039 169 169 169 288 288 288 F F F
183712024-02-08T06:46:40.718Z 2 040-059 167 167 167 284 284 284 F F F
183722024-02-08T06:46:40.718Z 3 060-079 167 167 167 284 284 284 F F F
183732024-02-08T06:46:40.718Z 4 080-099 169 169 169 288 288 288 F F F
183742024-02-08T06:46:40.718Z 5 100-119 168 168 168 286 286 286 F F F
183752024-02-08T06:46:40.719Z 6 120-139 167 167 167 284 284 284 F F F
183762024-02-08T06:46:40.742Z 7 140-159 162 162 162 277 277 277 F F F
183772024-02-08T06:46:40.742Z 8 160-179 162 162 162 277 277 277 F F F
183782024-02-08T06:46:40.743Z 9 180-199 163 163 163 278 278 278 F F F
183792024-02-08T06:46:40.743Z 10 200-219 162 162 162 277 277 277 F F F
183802024-02-08T06:46:40.743Z 11 220-239 168 168 168 286 286 286 F F F
183812024-02-08T06:46:40.743Z 12 240-259 165 165 165 282 282 282 F F F
183822024-02-08T06:46:40.743Z 13 260-279 169 169 169 288 288 288 F F F
183832024-02-08T06:46:40.743Z 14 280-299 164 164 164 281 281 281 F F F
183842024-02-08T06:46:40.743Z 15 300-319 169 169 169 288 288 288 F F F
183852024-02-08T06:46:40.743Z 16 320-339 165 165 165 282 282 282 F F F
183862024-02-08T06:46:40.744Z 17 340-359 168 168 168 286 286 286 F F F
183872024-02-08T06:46:40.744Z 18 360-379 169 169 169 288 288 288 F F F
183882024-02-08T06:46:40.744Z 19 380-399 169 169 169 288 288 288 F F F
183892024-02-08T06:46:40.744Z 20 400-419 164 164 164 281 281 281 F F F
183902024-02-08T06:46:40.744Z 21 420-439 169 169 169 288 288 288 F F F
183912024-02-08T06:46:40.745Z 22 440-459 167 167 167 284 284 284 F F F
183922024-02-08T06:46:40.745Z 23 460-479 169 169 169 288 288 288 F F F
183932024-02-08T06:46:40.745Z 24 480-499 164 164 164 281 281 281 F F F
183942024-02-08T06:46:40.745Z 25 500-519 160 160 160 274 274 274 F F F
183952024-02-08T06:46:40.745Z 26 520-539 168 168 168 286 286 286 F F F
183962024-02-08T06:46:40.745Z 27 540-559 169 169 169 288 288 288 F F F
183972024-02-08T06:46:40.745Z 28 560-579 169 169 169 288 288 288 F F F
183982024-02-08T06:46:40.745Z 29 580-599 166 166 166 283 283 283 F F F
183992024-02-08T06:46:40.746ZMax gen: 169, Max flush: 288
184002024-02-08T06:46:40.746Z
184012024-02-08T06:46:40.746ZBegin loop 85
184022024-02-08T06:46:40.746ZDownstairs to restart: 2
184032024-02-08T06:46:40.746Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538073586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818}
184042024-02-08T06:46:40.746ZCrucible runtime is spawned
184052024-02-08T06:46:40.747Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538407312Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818}
184062024-02-08T06:46:40.747Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538451138Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818}
184072024-02-08T06:46:40.747Z{"msg":"Crucible dcc8004a-9a06-46b4-a408-863f24579110 has session id: 3e350878-aa0c-488d-8011-9a774f3e4f26","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538471417Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184082024-02-08T06:46:40.747Z{"msg":"Upstairs opts: Upstairs UUID: dcc8004a-9a06-46b4-a408-863f24579110, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538517933Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184092024-02-08T06:46:40.747Z{"msg":"Crucible stats registered with UUID: dcc8004a-9a06-46b4-a408-863f24579110","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538545941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184102024-02-08T06:46:40.747Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538579979Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818}
184112024-02-08T06:46:40.747Z{"msg":"dcc8004a-9a06-46b4-a408-863f24579110 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538596727Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184122024-02-08T06:46:40.747Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538621816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184132024-02-08T06:46:40.747Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538662902Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184142024-02-08T06:46:40.748Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538686851Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184152024-02-08T06:46:40.748Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538729157Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184162024-02-08T06:46:40.748Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538775764Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184172024-02-08T06:46:40.748Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538809181Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184182024-02-08T06:46:40.748Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538838659Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"io task","client":"0","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184192024-02-08T06:46:40.748Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538889295Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"io task","client":"1","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184202024-02-08T06:46:40.748Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538940441Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184212024-02-08T06:46:40.749Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.538988338Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"io task","client":"1","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184222024-02-08T06:46:40.749Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.539027735Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"io task","client":"0","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184232024-02-08T06:46:40.749Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:39.539120398Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184242024-02-08T06:46:40.749Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:39.539155445Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184252024-02-08T06:46:40.749Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:39.539268927Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184262024-02-08T06:46:40.749Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.539296185Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184272024-02-08T06:46:40.749Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.539311264Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184282024-02-08T06:46:40.750Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.539401057Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184292024-02-08T06:46:40.750Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.539794167Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184302024-02-08T06:46:40.750Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.539832865Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184312024-02-08T06:46:40.750Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.539859463Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184322024-02-08T06:46:40.750Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.53988753Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184332024-02-08T06:46:40.750Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.540322268Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184342024-02-08T06:46:40.751Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.540437739Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184352024-02-08T06:46:40.751Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.540803772Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184362024-02-08T06:46:40.751Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.540883586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184372024-02-08T06:46:40.751Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.540930512Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184382024-02-08T06:46:40.751Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.54096225Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184392024-02-08T06:46:40.751Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.540978199Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184402024-02-08T06:46:40.751Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:39.540999027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184412024-02-08T06:46:41.327Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.54072415Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184422024-02-08T06:46:41.328Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.540874158Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184432024-02-08T06:46:41.333Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.542081418Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184442024-02-08T06:46:41.333Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.542126195Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184452024-02-08T06:46:41.333Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.542634276Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184462024-02-08T06:46:41.333Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543185095Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184472024-02-08T06:46:41.333Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543235931Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184482024-02-08T06:46:41.333Z{"msg":"[0]R flush_numbers[0..12]: [288, 288, 284, 284, 288, 286, 284, 277, 277, 278, 277, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543260479Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184492024-02-08T06:46:41.333Z{"msg":"[0]R generation[0..12]: [169, 169, 167, 167, 169, 168, 167, 162, 162, 163, 162, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543286737Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184502024-02-08T06:46:41.333Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543302036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184512024-02-08T06:46:41.333Z{"msg":"[1]R flush_numbers[0..12]: [288, 288, 284, 284, 288, 286, 284, 277, 277, 278, 277, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543322895Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184522024-02-08T06:46:41.333Z{"msg":"[1]R generation[0..12]: [169, 169, 167, 167, 169, 168, 167, 162, 162, 163, 162, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543337684Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184532024-02-08T06:46:41.334Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543378241Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184542024-02-08T06:46:41.334Z{"msg":"[2]R flush_numbers[0..12]: [288, 288, 284, 284, 288, 286, 284, 277, 277, 278, 277, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543393099Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184552024-02-08T06:46:41.334Z{Wait for a query_work_queue command to finish before sending IO
184562024-02-08T06:46:41.334Z"msg":"[2]R generation[0..12]: [169, 169, 167, 167, 169, 168, 167, 162, 162, 163, 162, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543416828Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184572024-02-08T06:46:41.334Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543431297Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184582024-02-08T06:46:41.334Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543451135Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184592024-02-08T06:46:41.334Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543465374Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184602024-02-08T06:46:41.334Z{"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543485122Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184612024-02-08T06:46:41.334Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543498821Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184622024-02-08T06:46:41.334Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.54351895Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184632024-02-08T06:46:41.334Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543533049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184642024-02-08T06:46:41.334Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543553537Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184652024-02-08T06:46:41.334Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543567306Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184662024-02-08T06:46:41.334Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543585375Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184672024-02-08T06:46:41.335Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543618992Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184682024-02-08T06:46:41.335Z{"msg":"dcc8004a-9a06-46b4-a408-863f24579110 is now active with session: 3e350878-aa0c-488d-8011-9a774f3e4f26","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543641711Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184692024-02-08T06:46:41.335Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543670888Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818,"session_id":"3e350878-aa0c-488d-8011-9a774f3e4f26"}
184702024-02-08T06:46:41.335Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:40.543688647Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818}
184712024-02-08T06:46:41.335ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184722024-02-08T06:46:41.335ZLoading write count information from file "/tmp/test_repair_verify.data"
184732024-02-08T06:46:41.335ZRun Repair workload
184742024-02-08T06:46:41.335Z01/30 Write block 13 len 32768 data: 12 11 11 10 10 11 12 13
184752024-02-08T06:46:41.339Z02/30 Write block 224 len 20480 data: 18 17 17 18 19
184762024-02-08T06:46:41.339Z03/30 Read block 204 len 16384
184772024-02-08T06:46:41.339Z04/30 Read block 216 len 12288
184782024-02-08T06:46:41.339Z05/30 Read block 198 len 20480
184792024-02-08T06:46:41.339Z06/30 Read block 103 len 40960
184802024-02-08T06:46:41.340Z07/30 Flush
184812024-02-08T06:46:41.343Z08/30 Write block 211 len 32768 data: 19 21 19 18 16 17 17 15
184822024-02-08T06:46:41.343Z09/30 Write block 537 len 16384 data: 13 12 11 11
184832024-02-08T06:46:41.343Z10/30 Read block 137 len 40960
184842024-02-08T06:46:41.346Z11/30 Flush
184852024-02-08T06:46:41.346Z12/30 Write block 538 len 4096 data: 13
184862024-02-08T06:46:41.346Z13/30 Read block 499 len 20480
184872024-02-08T06:46:41.349Z14/30 Flush
184882024-02-08T06:46:41.349Z15/30 Write block 379 len 24576 data: 11 11 8 11 10 14
184892024-02-08T06:46:41.349Z16/30 Read block 553 len 16384
184902024-02-08T06:46:41.352Z17/30 Write block 346 len 40960 data: 16 13 12 13 13 12 12 11 11 11
184912024-02-08T06:46:41.352Z18/30 Flush
184922024-02-08T06:46:41.356Z19/30 Write block 371 len 32768 data: 15 19 18 10 13 14 12 12
184932024-02-08T06:46:41.356Z20/30 Write block 342 len 4096 data: 19
184942024-02-08T06:46:41.356Z21/30 Read block 101 len 24576
184952024-02-08T06:46:41.356Z22/30 Write block 549 len 16384 data: 26 26 26 24
184962024-02-08T06:46:41.360Z23/30 Read block 100 len 20480
184972024-02-08T06:46:41.360Z24/30 Write block 244 len 8192 data: 14 14
184982024-02-08T06:46:41.360Z25/30 Read block 335 len 28672
184992024-02-08T06:46:41.360Z26/30 Write block 109 len 32768 data: 12 8 9 10 15 15 16 17
185002024-02-08T06:46:41.360Z27/30 Write block 325 len 8192 data: 16 14
185012024-02-08T06:46:41.364Z28/30 Read block 428 len 32768
185022024-02-08T06:46:41.365Z29/30 Write block 510 len 28672 data: 13 10 9 11 9 8 9
185032024-02-08T06:46:41.365Z30/30 Read block 170 len 40960
185042024-02-08T06:46:41.365Z----------------------------------------------------------------
185052024-02-08T06:46:41.365Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
185062024-02-08T06:46:41.365ZStates: Active Active Active
185072024-02-08T06:46:41.365ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185082024-02-08T06:46:41.365Z 1 Acked 1000 Write 8 Done Done Done false
185092024-02-08T06:46:41.365Z 2 Acked 1001 Write 5 Done Done Done false
185102024-02-08T06:46:41.365Z 3 Acked 1002 Read 4 Done Done Done false
185112024-02-08T06:46:41.366Z 4 Acked 1003 Read 3 Done Done Sent false
185122024-02-08T06:46:41.366Z 5 Acked 1004 Read 5 Done Done Sent false
185132024-02-08T06:46:41.366Z 6 Acked 1005 Read 10 Done Done Sent false
185142024-02-08T06:46:41.366Z 7 Acked 1006 Flush 0 Done Done Sent false
185152024-02-08T06:46:41.366Z 8 Acked 1007 Write 8 Done Done Sent false
185162024-02-08T06:46:41.366Z 9 Acked 1008 Write 4 Done Done Sent false
185172024-02-08T06:46:41.366Z 10 Acked 1009 Read 10 Done Done Sent false
185182024-02-08T06:46:41.366Z 11 Acked 1010 Flush 0 Done Done Sent false
185192024-02-08T06:46:41.366Z 12 Acked 1011 Write 1 Done Done Sent false
185202024-02-08T06:46:41.367Z 13 Acked 1012 Read 5 Done Done Sent false
185212024-02-08T06:46:41.367Z 14 Acked 1013 Flush 0 Done Done Sent false
185222024-02-08T06:46:41.367Z 15 Acked 1014 Write 6 Done Done Sent false
185232024-02-08T06:46:41.367Z 16 Acked 1015 Read 4 Done Done Sent false
185242024-02-08T06:46:41.367Z 17 Acked 1016 Write 10 Done Done Sent false
185252024-02-08T06:46:41.367Z 18 Acked 1017 Flush 0 Done Done Sent false
185262024-02-08T06:46:41.367Z 19 Acked 1018 Write 8 Done Done Sent false
185272024-02-08T06:46:41.367Z 20 Acked 1019 Write 1 Done Done Sent false
185282024-02-08T06:46:41.368Z 21 Acked 1020 Read 6 Done Done Sent false
185292024-02-08T06:46:41.368Z 22 Acked 1021 Write 4 Done Done Sent false
185302024-02-08T06:46:41.368Z 23 Acked 1022 Read 5 Done Done Sent false
185312024-02-08T06:46:41.368Z 24 Acked 1023 Write 2 Done Done Sent false
185322024-02-08T06:46:41.368Z 25 Acked 1024 Read 7 Done Done Sent false
185332024-02-08T06:46:41.368Z 26 Acked 1025 Write 8 Done Done Sent false
185342024-02-08T06:46:41.368Z 27 Acked 1026 Write 2 Done Done Sent false
185352024-02-08T06:46:41.368Z 28 Acked 1027 Read 8 Done Done Sent false
185362024-02-08T06:46:41.368Z 29 Acked 1028 Write 7 Done Done Sent false
185372024-02-08T06:46:41.369Z 30 Acked 1029 Read 10 Done Done Sent false
185382024-02-08T06:46:41.369Z STATES DS:0 DS:1 DS:2 TOTAL
185392024-02-08T06:46:41.369Z New 0 0 0 0
185402024-02-08T06:46:41.369Z Sent 0 0 27 27
185412024-02-08T06:46:41.369Z Done 30 30 3 63
185422024-02-08T06:46:41.369Z Skipped 0 0 0 0
185432024-02-08T06:46:41.369Z Error 0 0 0 0
185442024-02-08T06:46:41.369ZLast Flush: 1017 1017 0
185452024-02-08T06:46:41.370ZDownstairs last five completed:
185462024-02-08T06:46:41.370ZUpstairs last five completed: 30 29 28 27 26
185472024-02-08T06:46:41.370Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:40.577102728Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1818}
185482024-02-08T06:46:41.370ZWrote out file "/tmp/test_repair_verify.data"
185492024-02-08T06:46:41.373Z
185502024-02-08T06:46:41.374ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185512024-02-08T06:46:41.391ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185522024-02-08T06:46:41.391Z 0 000-019 170 170 169 289 289 288 F F T <---
185532024-02-08T06:46:41.391Z 1 020-039 170 170 169 289 289 288 F F T <---
185542024-02-08T06:46:41.391Z 2 040-059 167 167 167 284 284 284 F F F
185552024-02-08T06:46:41.391Z 3 060-079 167 167 167 284 284 284 F F F
185562024-02-08T06:46:41.391Z 4 080-099 169 169 169 288 288 288 F F F
185572024-02-08T06:46:41.391Z 5 100-119 168 168 168 286 286 286 T T F <---
185582024-02-08T06:46:41.391Z 6 120-139 167 167 167 284 284 284 F F F
185592024-02-08T06:46:41.392Z 7 140-159 162 162 162 277 277 277 F F F
185602024-02-08T06:46:41.392Z 8 160-179 162 162 162 277 277 277 F F F
185612024-02-08T06:46:41.395Z 9 180-199 163 163 163 278 278 278 F F F
185622024-02-08T06:46:41.395Z 10 200-219 170 170 162 290 290 277 F F F <---
185632024-02-08T06:46:41.396Z 11 220-239 170 170 168 289 289 286 F F T <---
185642024-02-08T06:46:41.396Z 12 240-259 165 165 165 282 282 282 T T F <---
185652024-02-08T06:46:41.396Z 13 260-279 169 169 169 288 288 288 F F F
185662024-02-08T06:46:41.396Z 14 280-299 164 164 164 281 281 281 F F F
185672024-02-08T06:46:41.396Z 15 300-319 169 169 169 288 288 288 F F F
185682024-02-08T06:46:41.396Z 16 320-339 165 165 165 282 282 282 T T F <---
185692024-02-08T06:46:41.396Z 17 340-359 170 170 168 292 292 286 T T F <---
185702024-02-08T06:46:41.396Z 18 360-379 170 170 169 292 292 288 T T F <---
185712024-02-08T06:46:41.397Z 19 380-399 170 170 169 292 292 288 F F F <---
185722024-02-08T06:46:41.397Z 20 400-419 164 164 164 281 281 281 F F F
185732024-02-08T06:46:41.397Z 21 420-439 169 169 169 288 288 288 F F F
185742024-02-08T06:46:41.397Z 22 440-459 167 167 167 284 284 284 F F F
185752024-02-08T06:46:41.397Z 23 460-479 169 169 169 288 288 288 F F F
185762024-02-08T06:46:41.397Z 24 480-499 164 164 164 281 281 281 F F F
185772024-02-08T06:46:41.397Z 25 500-519 160 160 160 274 274 274 T T F <---
185782024-02-08T06:46:41.397Z 26 520-539 170 170 168 291 291 286 F F F <---
185792024-02-08T06:46:41.398Z 27 540-559 170 170 169 290 290 288 T T F <---
185802024-02-08T06:46:41.398Z 28 560-579 169 169 169 288 288 288 F F F
185812024-02-08T06:46:41.398Z 29 580-599 166 166 166 283 283 283 F F F
185822024-02-08T06:46:41.398ZMax gen: 170, Max flush: 292
185832024-02-08T06:46:41.398ZError: Difference in extent metadata found!
185842024-02-08T06:46:41.398ZA Difference in extent metadata is expected here
185852024-02-08T06:46:41.398ZOn loop 85
185862024-02-08T06:46:41.398Z
185872024-02-08T06:46:41.402ZVerifying data now
185882024-02-08T06:46:47.060ZNo change after verify
185892024-02-08T06:46:47.060ZLoop: 85 Downstairs dump after verify (and repair):
185902024-02-08T06:46:47.074ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185912024-02-08T06:46:47.075Z 0 000-019 170 170 170 289 289 289 F F F
185922024-02-08T06:46:47.075Z 1 020-039 170 170 170 289 289 289 F F F
185932024-02-08T06:46:47.075Z 2 040-059 167 167 167 284 284 284 F F F
185942024-02-08T06:46:47.075Z 3 060-079 167 167 167 284 284 284 F F F
185952024-02-08T06:46:47.075Z 4 080-099 169 169 169 288 288 288 F F F
185962024-02-08T06:46:47.078Z 5 100-119 171 171 171 293 293 293 F F F
185972024-02-08T06:46:47.078Z 6 120-139 167 167 167 284 284 284 F F F
185982024-02-08T06:46:47.078Z 7 140-159 162 162 162 277 277 277 F F F
185992024-02-08T06:46:47.078Z 8 160-179 162 162 162 277 277 277 F F F
186002024-02-08T06:46:47.078Z 9 180-199 163 163 163 278 278 278 F F F
186012024-02-08T06:46:47.079Z 10 200-219 170 170 170 290 290 290 F F F
186022024-02-08T06:46:47.079Z 11 220-239 170 170 170 289 289 289 F F F
186032024-02-08T06:46:47.079Z 12 240-259 171 171 171 293 293 293 F F F
186042024-02-08T06:46:47.079Z 13 260-279 169 169 169 288 288 288 F F F
186052024-02-08T06:46:47.079Z 14 280-299 164 164 164 281 281 281 F F F
186062024-02-08T06:46:47.079Z 15 300-319 169 169 169 288 288 288 F F F
186072024-02-08T06:46:47.079Z 16 320-339 171 171 171 293 293 293 F F F
186082024-02-08T06:46:47.079Z 17 340-359 171 171 171 293 293 293 F F F
186092024-02-08T06:46:47.079Z 18 360-379 171 171 171 293 293 293 F F F
186102024-02-08T06:46:47.079Z 19 380-399 170 170 170 292 292 292 F F F
186112024-02-08T06:46:47.079Z 20 400-419 164 164 164 281 281 281 F F F
186122024-02-08T06:46:47.079Z 21 420-439 169 169 169 288 288 288 F F F
186132024-02-08T06:46:47.079Z 22 440-459 167 167 167 284 284 284 F F F
186142024-02-08T06:46:47.079Z 23 460-479 169 169 169 288 288 288 F F F
186152024-02-08T06:46:47.079Z 24 480-499 164 164 164 281 281 281 F F F
186162024-02-08T06:46:47.079Z 25 500-519 171 171 171 293 293 293 F F F
186172024-02-08T06:46:47.080Z 26 520-539 170 170 170 291 291 291 F F F
186182024-02-08T06:46:47.080Z 27 540-559 171 171 171 293 293 293 F F F
186192024-02-08T06:46:47.080Z 28 560-579 169 169 169 288 288 288 F F F
186202024-02-08T06:46:47.080Z 29 580-599 166 166 166 283 283 283 F F F
186212024-02-08T06:46:47.080ZMax gen: 171, Max flush: 293
186222024-02-08T06:46:47.080Z
186232024-02-08T06:46:47.080ZBegin loop 86
186242024-02-08T06:46:47.080ZDownstairs to restart: 1
186252024-02-08T06:46:47.095Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308169532Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826}
186262024-02-08T06:46:47.095Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308443062Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826}
186272024-02-08T06:46:47.100Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.30846837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826}
186282024-02-08T06:46:47.100Z{"msg":"Crucible a8640d74-dd09-4274-98e8-fe01387fc630 has session id: c3d1943f-8f36-437e-9c39-95fd38eecf16",Crucible runtime is spawned
186292024-02-08T06:46:47.100Z"v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308583711Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186302024-02-08T06:46:47.100Z{"msg":"Upstairs opts: Upstairs UUID: a8640d74-dd09-4274-98e8-fe01387fc630, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308619158Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186312024-02-08T06:46:47.101Z{"msg":"Crucible stats registered with UUID: a8640d74-dd09-4274-98e8-fe01387fc630","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308653136Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186322024-02-08T06:46:47.101Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308676794Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826}
186332024-02-08T06:46:47.101Z{"msg":"a8640d74-dd09-4274-98e8-fe01387fc630 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308753238Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186342024-02-08T06:46:47.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308778047Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186352024-02-08T06:46:47.101Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308798735Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186362024-02-08T06:46:47.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308822143Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186372024-02-08T06:46:47.101Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308848881Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186382024-02-08T06:46:47.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308875449Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186392024-02-08T06:46:47.101Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308917716Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186402024-02-08T06:46:47.101Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308946504Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186412024-02-08T06:46:47.101Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.308970292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186422024-02-08T06:46:47.101Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.30899457Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186432024-02-08T06:46:47.101Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.309152068Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186442024-02-08T06:46:47.101Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:46.309211394Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186452024-02-08T06:46:47.101Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:46.309244381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186462024-02-08T06:46:47.101Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.309286538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186472024-02-08T06:46:47.102Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:46.309332295Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186482024-02-08T06:46:47.102Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.309373262Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186492024-02-08T06:46:47.102Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.3093961Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186502024-02-08T06:46:47.102Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.309418208Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186512024-02-08T06:46:47.102Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.309768552Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186522024-02-08T06:46:47.102Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.30979702Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186532024-02-08T06:46:47.102Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.309813639Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186542024-02-08T06:46:47.102Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.309846696Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186552024-02-08T06:46:47.102Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.310357168Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186562024-02-08T06:46:47.102Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.310491018Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186572024-02-08T06:46:47.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.310901897Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186582024-02-08T06:46:47.102Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.310939434Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186592024-02-08T06:46:47.102Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.310961292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186602024-02-08T06:46:47.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.310975551Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186612024-02-08T06:46:47.102Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.311017478Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186622024-02-08T06:46:47.103Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:46.311039757Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186632024-02-08T06:46:48.098Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.310111668Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186642024-02-08T06:46:48.098Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.310273736Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186652024-02-08T06:46:48.106Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.311188417Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186662024-02-08T06:46:48.106Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.311266851Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186672024-02-08T06:46:48.106Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.311736586Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186682024-02-08T06:46:48.106Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312281466Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186692024-02-08T06:46:48.106Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312327172Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186702024-02-08T06:46:48.106Z{"msg":"[0]R flush_numbers[0..12]: [289, 289, 284, 284, 288, 293, 284, 277, 277, 278, 290, 289]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312363859Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186712024-02-08T06:46:48.106Z{"msg":"[0]R generation[0..12]: [170, 170, 167, 167, 169, 171, 167, 162, 162, 163, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312399757Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186722024-02-08T06:46:48.106Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312436434Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186732024-02-08T06:46:48.106Z{"msg":"[1]R flush_numbers[0..12]: [289, 289, 284, 284, 288, 293, 284, 277, 277, 278, 290, 289]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312471871Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186742024-02-08T06:46:48.107Z{"msg":"[1]R generation[0..12]: [170, 170, 167, 167, 169, 171, 167, 162, 162, 163, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312506898Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186752024-02-08T06:46:48.107ZWait for a query_work_queue command to finish before sending IO
186762024-02-08T06:46:48.107Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312553615Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186772024-02-08T06:46:48.107Z{"msg":"[2]R flush_numbers[0..12]: [289, 289, 284, 284, 288, 293, 284, 277, 277, 278, 290, 289]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312593572Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186782024-02-08T06:46:48.107Z{"msg":"[2]R generation[0..12]: [170, 170, 167, 167, 169, 171, 167, 162, 162, 163, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312632349Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186792024-02-08T06:46:48.107Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312670906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186802024-02-08T06:46:48.107Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312697524Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186812024-02-08T06:46:48.107Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312735681Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186822024-02-08T06:46:48.107Z{"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312773938Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186832024-02-08T06:46:48.107Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312811656Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186842024-02-08T06:46:48.107Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312847733Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186852024-02-08T06:46:48.107Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.31288175Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186862024-02-08T06:46:48.107Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312918128Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186872024-02-08T06:46:48.107Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312957405Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186882024-02-08T06:46:48.107Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.312976673Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186892024-02-08T06:46:48.108Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
186902024-02-08T06:46:48.108Z,"time":"2024-02-08T06:46:47.31302563Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186912024-02-08T06:46:48.108Z{"msg":"a8640d74-dd09-4274-98e8-fe01387fc630 is now active with session: c3d1943f-8f36-437e-9c39-95fd38eecf16","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.313042088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186922024-02-08T06:46:48.108Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.313054797Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826,"session_id":"c3d1943f-8f36-437e-9c39-95fd38eecf16"}
186932024-02-08T06:46:48.108Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:47.313067276Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826}
186942024-02-08T06:46:48.108ZLoading write count information from file "/tmp/test_repair_verify.data"
186952024-02-08T06:46:48.108ZRun Repair workload
186962024-02-08T06:46:48.108Z01/30 Flush
186972024-02-08T06:46:48.112Z02/30 Write block 56 len 8192 data: 14 15
186982024-02-08T06:46:48.112Z03/30 Write block 112 len 4096 data: 11
186992024-02-08T06:46:48.112Z04/30 Read block 592 len 16384
187002024-02-08T06:46:48.112Z05/30 Write block 539 len 40960 data: 12 12 11 9 11 11 12 16 20 24
187012024-02-08T06:46:48.112Z06/30 Write block 54 len 12288 data: 11 11 15
187022024-02-08T06:46:48.112Z07/30 Read block 100 len 28672
187032024-02-08T06:46:48.112Z08/30 Write block 280 len 24576 data: 15 13 11 10 12 11
187042024-02-08T06:46:48.112Z09/30 Read block 370 len 24576
187052024-02-08T06:46:48.112Z10/30 Read block 106 len 40960
187062024-02-08T06:46:48.112Z11/30 Read block 436 len 8192
187072024-02-08T06:46:48.112Z12/30 Read block 257 len 16384
187082024-02-08T06:46:48.112Z13/30 Read block 12 len 12288
187092024-02-08T06:46:48.121Z14/30 Write block 384 len 24576 data: 15 11 11 10 12 15
187102024-02-08T06:46:48.121Z15/30 Write block 319 len 16384 data: 11 16 14 16
187112024-02-08T06:46:48.122Z16/30 Read block 191 len 12288
187122024-02-08T06:46:48.122Z17/30 Write block 211 len 36864 data: 20 22 20 19 17 18 18 16 17
187132024-02-08T06:46:48.122Z18/30 Write block 439 len 24576 data: 12 14 14 13 14 14
187142024-02-08T06:46:48.122Z19/30 Read block 347 len 32768
187152024-02-08T06:46:48.550Z20/30 Write block 135 len 20480 data: 9 10 14 14 12
187162024-02-08T06:46:48.550Z21/30 Write block 484 len 8192 data: 12 13
187172024-02-08T06:46:48.550Z22/30 Read block 240 len 12288
187182024-02-08T06:46:48.550Z23/30 Read block 538 len 40960
187192024-02-08T06:46:48.550Z24/30 Read block 431 len 36864
187202024-02-08T06:46:48.551Z25/30 Flush
187212024-02-08T06:46:48.551Z26/30 Write block 488 len 12288 data: 17 15 16
187222024-02-08T06:46:48.551Z27/30 Read block 130 len 36864
187232024-02-08T06:46:48.551Z28/30 Read block 154 len 16384
187242024-02-08T06:46:48.552Z29/30 Read block 497 len 8192
187252024-02-08T06:46:48.552Z30/30 Write block 147 len 32768 data: 8 8 11 9 12 11 13 15
187262024-02-08T06:46:48.552Z----------------------------------------------------------------
187272024-02-08T06:46:48.552Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:29
187282024-02-08T06:46:48.552ZStates: Active Active Active
187292024-02-08T06:46:48.552ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187302024-02-08T06:46:48.552Z 2 Acked 1001 Write 2 Done Done Done false
187312024-02-08T06:46:48.552Z 3 Acked 1002 Write 1 Done Done Done false
187322024-02-08T06:46:48.552Z 4 Acked 1003 Read 4 Done Done Done false
187332024-02-08T06:46:48.552Z 5 Acked 1004 Write 10 Done Sent Done false
187342024-02-08T06:46:48.553Z 6 Acked 1005 Write 3 Done Sent Done false
187352024-02-08T06:46:48.553Z 7 Acked 1006 Read 7 Done Sent Done false
187362024-02-08T06:46:48.553Z 8 Acked 1007 Write 6 Done Sent Done false
187372024-02-08T06:46:48.553Z 9 Acked 1008 Read 6 Done Sent Done false
187382024-02-08T06:46:48.553Z 10 Acked 1009 Read 10 Done Sent Done false
187392024-02-08T06:46:48.553Z 11 Acked 1010 Read 2 Done Sent Done false
187402024-02-08T06:46:48.553Z 12 Acked 1011 Read 4 Done Sent Done false
187412024-02-08T06:46:48.553Z 13 Acked 1012 Read 3 Done Sent Done false
187422024-02-08T06:46:48.553Z 14 Acked 1013 Write 6 Done Sent Done false
187432024-02-08T06:46:48.553Z 15 Acked 1014 Write 4 Done Sent Done false
187442024-02-08T06:46:48.553Z 16 Acked 1015 Read 3 Done Sent Done false
187452024-02-08T06:46:48.553Z 17 Acked 1016 Write 9 Done Sent Done false
187462024-02-08T06:46:48.553Z 18 Acked 1017 Write 6 Done Sent Done false
187472024-02-08T06:46:48.554Z 19 Acked 1018 Read 8 Done Sent Done false
187482024-02-08T06:46:48.554Z 20 Acked 1019 Write 5 Done Sent Done false
187492024-02-08T06:46:48.554Z 21 Acked 1020 Write 2 Done Sent Done false
187502024-02-08T06:46:48.554Z 22 Acked 1021 Read 3 Done Sent Done false
187512024-02-08T06:46:48.554Z 23 Acked 1022 Read 10 Done Sent Done false
187522024-02-08T06:46:48.554Z 24 Acked 1023 Read 9 Done Sent Done false
187532024-02-08T06:46:48.554Z 25 Acked 1024 Flush 0 Done Sent Done false
187542024-02-08T06:46:48.554Z 26 Acked 1025 Write 3 Done Sent Done false
187552024-02-08T06:46:48.554Z 27 Acked 1026 Read 9 Done Sent Done false
187562024-02-08T06:46:48.554Z 28 Acked 1027 Read 4 Done Sent Done false
187572024-02-08T06:46:48.554Z 29 Acked 1028 Read 2 Done Sent Done false
187582024-02-08T06:46:48.554Z 30 Acked 1029 Write 8 Sent Sent Sent false
187592024-02-08T06:46:48.554Z STATES DS:0 DS:1 DS:2 TOTAL
187602024-02-08T06:46:48.554Z New 0 0 0 0
187612024-02-08T06:46:48.554Z Sent 1 26 1 28
187622024-02-08T06:46:48.555Z Done 28 3 28 59
187632024-02-08T06:46:48.555Z Skipped 0 0 0 0
187642024-02-08T06:46:48.555Z Error 0 0 0 0
187652024-02-08T06:46:48.555ZLast Flush: 1024 1000 1024
187662024-02-08T06:46:48.555ZDownstairs last five completed: 1000
187672024-02-08T06:46:48.555ZUpstairs last five completed: 30 29 28 27 26
187682024-02-08T06:46:48.555Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:47.343287278Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1826}
187692024-02-08T06:46:48.555ZWrote out file "/tmp/test_repair_verify.data"
187702024-02-08T06:46:48.555Z
187712024-02-08T06:46:48.555ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
187722024-02-08T06:46:48.555ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187732024-02-08T06:46:48.555Z 0 000-019 170 170 170 289 289 289 F F F
187742024-02-08T06:46:48.555Z 1 020-039 170 170 170 289 289 289 F F F
187752024-02-08T06:46:48.555Z 2 040-059 172 167 172 295 284 295 F T F <---
187762024-02-08T06:46:48.556Z 3 060-079 167 167 167 284 284 284 F F F
187772024-02-08T06:46:48.556Z 4 080-099 169 169 169 288 288 288 F F F
187782024-02-08T06:46:48.556Z 5 100-119 172 171 172 295 293 295 F T F <---
187792024-02-08T06:46:48.556Z 6 120-139 172 167 172 295 284 295 F F F <---
187802024-02-08T06:46:48.556Z 7 140-159 162 162 162 277 277 277 T F T <---
187812024-02-08T06:46:48.556Z 8 160-179 162 162 162 277 277 277 F F F
187822024-02-08T06:46:48.556Z 9 180-199 163 163 163 278 278 278 F F F
187832024-02-08T06:46:48.556Z 10 200-219 172 170 172 295 290 295 F F F <---
187842024-02-08T06:46:48.556Z 11 220-239 170 170 170 289 289 289 F F F
187852024-02-08T06:46:48.556Z 12 240-259 171 171 171 293 293 293 F F F
187862024-02-08T06:46:48.556Z 13 260-279 169 169 169 288 288 288 F F F
187872024-02-08T06:46:48.556Z 14 280-299 172 164 172 295 281 295 F F F <---
187882024-02-08T06:46:48.556Z 15 300-319 172 169 172 295 288 295 F F F <---
187892024-02-08T06:46:48.556Z 16 320-339 172 171 172 295 293 295 F F F <---
187902024-02-08T06:46:48.556Z 17 340-359 171 171 171 293 293 293 F F F
187912024-02-08T06:46:48.556Z 18 360-379 171 171 171 293 293 293 F F F
187922024-02-08T06:46:48.557Z 19 380-399 172 170 172 295 292 295 F F F <---
187932024-02-08T06:46:48.557Z 20 400-419 164 164 164 281 281 281 F F F
187942024-02-08T06:46:48.557Z 21 420-439 172 169 172 295 288 295 F F F <---
187952024-02-08T06:46:48.557Z 22 440-459 172 167 172 295 284 295 F F F <---
187962024-02-08T06:46:48.557Z 23 460-479 169 169 169 288 288 288 F F F
187972024-02-08T06:46:48.557Z 24 480-499 172 164 172 295 281 295 T F T <---
187982024-02-08T06:46:48.557Z 25 500-519 171 171 171 293 293 293 F F F
187992024-02-08T06:46:48.557Z 26 520-539 172 170 172 295 291 295 F F F <---
188002024-02-08T06:46:48.557Z 27 540-559 172 171 172 295 293 295 F F F <---
188012024-02-08T06:46:48.557Z 28 560-579 169 169 169 288 288 288 F F F
188022024-02-08T06:46:48.557Z 29 580-599 166 166 166 283 283 283 F F F
188032024-02-08T06:46:48.557ZMax gen: 172, Max flush: 295
188042024-02-08T06:46:48.557ZError: Difference in extent metadata found!
188052024-02-08T06:46:48.557ZA Difference in extent metadata is expected here
188062024-02-08T06:46:48.557ZOn loop 86
188072024-02-08T06:46:48.557Z
188082024-02-08T06:46:48.558ZVerifying data now
188092024-02-08T06:46:53.827ZNo change after verify
188102024-02-08T06:46:53.827ZLoop: 86 Downstairs dump after verify (and repair):
188112024-02-08T06:46:53.845ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188122024-02-08T06:46:53.845Z 0 000-019 170 170 170 289 289 289 F F F
188132024-02-08T06:46:53.845Z 1 020-039 170 170 170 289 289 289 F F F
188142024-02-08T06:46:53.845Z 2 040-059 172 172 172 295 295 295 F F F
188152024-02-08T06:46:53.845Z 3 060-079 167 167 167 284 284 284 F F F
188162024-02-08T06:46:53.845Z 4 080-099 169 169 169 288 288 288 F F F
188172024-02-08T06:46:53.846Z 5 100-119 172 172 172 295 295 295 F F F
188182024-02-08T06:46:53.846Z 6 120-139 172 172 172 295 295 295 F F F
188192024-02-08T06:46:53.846Z 7 140-159 173 173 173 296 296 296 F F F
188202024-02-08T06:46:53.846Z 8 160-179 162 162 162 277 277 277 F F F
188212024-02-08T06:46:53.846Z 9 180-199 163 163 163 278 278 278 F F F
188222024-02-08T06:46:53.846Z 10 200-219 172 172 172 295 295 295 F F F
188232024-02-08T06:46:53.846Z 11 220-239 170 170 170 289 289 289 F F F
188242024-02-08T06:46:53.846Z 12 240-259 171 171 171 293 293 293 F F F
188252024-02-08T06:46:53.849Z 13 260-279 169 169 169 288 288 288 F F F
188262024-02-08T06:46:53.849Z 14 280-299 172 172 172 295 295 295 F F F
188272024-02-08T06:46:53.850Z 15 300-319 172 172 172 295 295 295 F F F
188282024-02-08T06:46:53.850Z 16 320-339 172 172 172 295 295 295 F F F
188292024-02-08T06:46:53.850Z 17 340-359 171 171 171 293 293 293 F F F
188302024-02-08T06:46:53.850Z 18 360-379 171 171 171 293 293 293 F F F
188312024-02-08T06:46:53.850Z 19 380-399 172 172 172 295 295 295 F F F
188322024-02-08T06:46:53.850Z 20 400-419 164 164 164 281 281 281 F F F
188332024-02-08T06:46:53.850Z 21 420-439 172 172 172 295 295 295 F F F
188342024-02-08T06:46:53.850Z 22 440-459 172 172 172 295 295 295 F F F
188352024-02-08T06:46:53.850Z 23 460-479 169 169 169 288 288 288 F F F
188362024-02-08T06:46:53.850Z 24 480-499 173 173 173 296 296 296 F F F
188372024-02-08T06:46:53.850Z 25 500-519 171 171 171 293 293 293 F F F
188382024-02-08T06:46:53.850Z 26 520-539 172 172 172 295 295 295 F F F
188392024-02-08T06:46:53.850Z 27 540-559 172 172 172 295 295 295 F F F
188402024-02-08T06:46:53.850Z 28 560-579 169 169 169 288 288 288 F F F
188412024-02-08T06:46:53.850Z 29 580-599 166 166 166 283 283 283 F F F
188422024-02-08T06:46:53.850ZMax gen: 173, Max flush: 296
188432024-02-08T06:46:53.851Z
188442024-02-08T06:46:53.851ZBegin loop 87
188452024-02-08T06:46:53.851ZDownstairs to restart: 2
188462024-02-08T06:46:53.862Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.074842965Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834}
188472024-02-08T06:46:53.863Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075117824Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834}
188482024-02-08T06:46:53.867Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075146272Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834}
188492024-02-08T06:46:53.867Z{"msg":"Crucible 9f4313af-278b-4044-a3a6-74554a262d97 has session id: 0a027ebb-d7c3-4829-ab17-1e6fe6d50e30","v":0,"name":"crucible","level":30Crucible runtime is spawned
188502024-02-08T06:46:53.867Z,"time":"2024-02-08T06:46:53.075204907Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188512024-02-08T06:46:53.867Z{"msg":"Upstairs opts: Upstairs UUID: 9f4313af-278b-4044-a3a6-74554a262d97, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075261793Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188522024-02-08T06:46:53.867Z{"msg":"Crucible stats registered with UUID: 9f4313af-278b-4044-a3a6-74554a262d97","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075285512Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188532024-02-08T06:46:53.868Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075320389Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834}
188542024-02-08T06:46:53.868Z{"msg":"9f4313af-278b-4044-a3a6-74554a262d97 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075383394Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188552024-02-08T06:46:53.868Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075427251Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188562024-02-08T06:46:53.868Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075449809Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188572024-02-08T06:46:53.868Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075480797Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188582024-02-08T06:46:53.868Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075516994Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188592024-02-08T06:46:53.868Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075547012Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188602024-02-08T06:46:53.868Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.07557303Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188612024-02-08T06:46:53.868Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075613427Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188622024-02-08T06:46:53.868Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075686971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"io task","client":"0","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188632024-02-08T06:46:53.868Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075720329Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"io task","client":"1","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188642024-02-08T06:46:53.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075845239Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188652024-02-08T06:46:53.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075895376Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"io task","client":"0","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188662024-02-08T06:46:53.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.075921424Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"io task","client":"1","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188672024-02-08T06:46:53.868Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.076568335Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188682024-02-08T06:46:53.869Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.076606722Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188692024-02-08T06:46:53.869Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.076623461Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188702024-02-08T06:46:53.869Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.076649129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188712024-02-08T06:46:53.869Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.07716848Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188722024-02-08T06:46:53.869Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.077352316Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188732024-02-08T06:46:53.869Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.077753736Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188742024-02-08T06:46:53.869Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.077805562Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188752024-02-08T06:46:53.869Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.077844529Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188762024-02-08T06:46:53.869Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.077888536Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188772024-02-08T06:46:53.869Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.077913434Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188782024-02-08T06:46:53.869Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.077927713Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188792024-02-08T06:46:53.869Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.077955501Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188802024-02-08T06:46:53.869Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.077993558Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188812024-02-08T06:46:53.874Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.078452824Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188822024-02-08T06:46:53.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.07889425Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188832024-02-08T06:46:53.874Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.078932108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188842024-02-08T06:46:53.874Z{"msg":"[0]R flush_numbers[0..12]: [289, 289, 295, 284, 288, 295, 295, 296, 277, 278, 295, 289]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.078949156Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188852024-02-08T06:46:53.874Z{"msg":"[0]R generation[0..12]: [170, 170, 172, 167, 169, 172, 172, 173, 162, 163, 172, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.078971345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188862024-02-08T06:46:53.874Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.078986054Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188872024-02-08T06:46:53.875Z{"msg":"[1]R flush_numbers[0..12]: [289, 289, 295, 284, 288, 295, 295, 296, 277, 278, 295, 289]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079022631Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188882024-02-08T06:46:53.875Z{"msg":"[1]R generation[0..12]: [170, 170, 172, 167, 169, 172, 172, 173, 162, 163, 172, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079049199Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188892024-02-08T06:46:53.875Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079064268Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188902024-02-08T06:46:53.875ZWait for a query_work_queue command to finish before sending IO
188912024-02-08T06:46:53.875Z{"msg":"[2]R flush_numbers[0..12]: [289, 289, 295, 284, 288, 295, 295, 296, 277, 278, 295, 289]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079090396Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188922024-02-08T06:46:53.875Z{"msg":"[2]R generation[0..12]: [170, 170, 172, 167, 169, 172, 172, 173, 162, 163, 172, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079105305Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188932024-02-08T06:46:53.875Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079125993Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188942024-02-08T06:46:53.875Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079139552Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188952024-02-08T06:46:53.875Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079159641Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188962024-02-08T06:46:53.875Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079173849Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188972024-02-08T06:46:53.875Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079206737Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188982024-02-08T06:46:53.875Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079238205Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
188992024-02-08T06:46:53.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079261963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
189002024-02-08T06:46:53.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079278412Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
189012024-02-08T06:46:53.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.07930106Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
189022024-02-08T06:46:53.876Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079314929Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
189032024-02-08T06:46:53.876Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079335557Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
189042024-02-08T06:46:53.876Z{"msg":"9f4313af-278b-4044-a3a6-74554a262d97 is now active with session: 0a027ebb-d7c3-4829-ab17-1e6fe6d50e30","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079356286Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
189052024-02-08T06:46:53.876Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079386884Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834,"session_id":"0a027ebb-d7c3-4829-ab17-1e6fe6d50e30"}
189062024-02-08T06:46:53.876Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:53.079419541Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834}
189072024-02-08T06:46:53.876ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189082024-02-08T06:46:53.876ZLoading write count information from file "/tmp/test_repair_verify.data"
189092024-02-08T06:46:53.876ZRun Repair workload
189102024-02-08T06:46:53.876Z01/30 Write block 547 len 32768 data: 21 25 27 27 27 25 22 19
189112024-02-08T06:46:53.876Z02/30 Write block 77 len 36864 data: 10 10 12 13 13 15 14 14 14
189122024-02-08T06:46:53.876Z03/30 Write block 552 len 20480 data: 26 23 20 19 22
189132024-02-08T06:46:53.876Z04/30 Write block 591 len 12288 data: 15 13 14
189142024-02-08T06:46:53.876Z05/30 Read block 210 len 8192
189152024-02-08T06:46:53.876Z06/30 Read block 371 len 12288
189162024-02-08T06:46:53.880Z07/30 Write block 335 len 8192 data: 17 19
189172024-02-08T06:46:53.880Z08/30 Read block 493 len 28672
189182024-02-08T06:46:53.880Z09/30 Read block 52 len 12288
189192024-02-08T06:46:53.880Z10/30 Write block 59 len 4096 data: 10
189202024-02-08T06:46:53.880Z11/30 Read block 308 len 12288
189212024-02-08T06:46:53.880Z12/30 Read block 255 len 24576
189222024-02-08T06:46:53.880Z13/30 Write block 543 len 28672 data: 12 12 13 17 22 26 28
189232024-02-08T06:46:53.880Z14/30 Write block 141 len 16384 data: 9 8 9 8
189242024-02-08T06:46:53.880Z15/30 Write block 551 len 16384 data: 28 27 24 21
189252024-02-08T06:46:53.880Z16/30 Read block 21 len 20480
189262024-02-08T06:46:53.883Z17/30 Read block 274 len 4096
189272024-02-08T06:46:53.883Z18/30 Read block 65 len 40960
189282024-02-08T06:46:53.886Z19/30 Write block 492 len 20480 data: 14 14 13 10 9
189292024-02-08T06:46:53.886Z20/30 Read block 494 len 28672
189302024-02-08T06:46:53.886Z21/30 Read block 367 len 40960
189312024-02-08T06:46:53.886Z22/30 Write block 423 len 8192 data: 12 11
189322024-02-08T06:46:53.892Z23/30 Write block 95 len 4096 data: 19
189332024-02-08T06:46:53.892Z24/30 Write block 202 len 40960 data: 15 14 14 16 16 17 14 14 12 21
189342024-02-08T06:46:53.892Z25/30 Read block 113 len 40960
189352024-02-08T06:46:53.892Z26/30 Read block 188 len 28672
189362024-02-08T06:46:53.896Z27/30 Read block 107 len 24576
189372024-02-08T06:46:53.896Z28/30 Write block 238 len 40960 data: 16 17 14 18 14 17 15 15 14 11
189382024-02-08T06:46:53.896Z29/30 Read block 122 len 32768
189392024-02-08T06:46:53.896Z30/30 Write block 20 len 28672 data: 14 11 12 12 16 17 18
189402024-02-08T06:46:53.896Z----------------------------------------------------------------
189412024-02-08T06:46:53.896Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
189422024-02-08T06:46:53.896ZStates: Active Active Active
189432024-02-08T06:46:53.896ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189442024-02-08T06:46:53.896Z 1 Acked 1000 Write 8 Done Done Sent false
189452024-02-08T06:46:53.896Z 2 Acked 1001 Write 9 Done Done Sent false
189462024-02-08T06:46:53.896Z 3 Acked 1002 Write 5 Done Done Sent false
189472024-02-08T06:46:53.897Z 4 Acked 1003 Write 3 Done Done Sent false
189482024-02-08T06:46:53.897Z 5 Acked 1004 Read 2 Done Done Sent false
189492024-02-08T06:46:53.897Z 6 Acked 1005 Read 3 Done Done Sent false
189502024-02-08T06:46:53.897Z 7 Acked 1006 Write 2 Done Done Sent false
189512024-02-08T06:46:53.897Z 8 Acked 1007 Read 7 Done Done Sent false
189522024-02-08T06:46:53.897Z 9 Acked 1008 Read 3 Done Done Sent false
189532024-02-08T06:46:53.897Z 10 Acked 1009 Write 1 Done Done Sent false
189542024-02-08T06:46:53.897Z 11 Acked 1010 Read 3 Done Done Sent false
189552024-02-08T06:46:53.897Z 12 Acked 1011 Read 6 Done Done Sent false
189562024-02-08T06:46:53.897Z 13 Acked 1012 Write 7 Done Done Sent false
189572024-02-08T06:46:53.897Z 14 Acked 1013 Write 4 Done Done Sent false
189582024-02-08T06:46:53.897Z 15 Acked 1014 Write 4 Done Done Sent false
189592024-02-08T06:46:53.897Z 16 Acked 1015 Read 5 Done Done Sent false
189602024-02-08T06:46:53.897Z 17 Acked 1016 Read 1 Done Done Sent false
189612024-02-08T06:46:53.897Z 18 Acked 1017 Read 10 Done Done Sent false
189622024-02-08T06:46:53.898Z 19 Acked 1018 Write 5 Done Done Sent false
189632024-02-08T06:46:53.898Z 20 Acked 1019 Read 7 Done Done Sent false
189642024-02-08T06:46:53.898Z 21 Acked 1020 Read 10 Done Done Sent false
189652024-02-08T06:46:53.898Z 22 Acked 1021 Write 2 Done Done Sent false
189662024-02-08T06:46:53.898Z 23 Acked 1022 Write 1 Done Done Sent false
189672024-02-08T06:46:53.898Z 24 Acked 1023 Write 10 Done Done Sent false
189682024-02-08T06:46:53.898Z 25 Acked 1024 Read 10 Done Done Sent false
189692024-02-08T06:46:53.898Z 26 Acked 1025 Read 7 Done Done Sent false
189702024-02-08T06:46:53.898Z 27 Acked 1026 Read 6 Done Done Sent false
189712024-02-08T06:46:53.898Z 28 Acked 1027 Write 10 Done Done Sent false
189722024-02-08T06:46:53.898Z 29 Acked 1028 Read 8 Done Done Sent false
189732024-02-08T06:46:53.898Z 30 Acked 1029 Write 7 Sent Sent Sent false
189742024-02-08T06:46:53.898Z STATES DS:0 DS:1 DS:2 TOTAL
189752024-02-08T06:46:53.898Z New 0 0 0 0
189762024-02-08T06:46:53.898Z Sent 1 1 30 32
189772024-02-08T06:46:53.898Z Done 29 29 0 58
189782024-02-08T06:46:53.899Z Skipped 0 0 0 0
189792024-02-08T06:46:53.899Z Error 0 0 0 0
189802024-02-08T06:46:53.899ZLast Flush: 0 0 0
189812024-02-08T06:46:53.899ZDownstairs last five completed:
189822024-02-08T06:46:53.899ZUpstairs last five completed: 30 29 28 27 26
189832024-02-08T06:46:53.899Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:53.106904008Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1834}
189842024-02-08T06:46:53.899ZWrote out file "/tmp/test_repair_verify.data"
189852024-02-08T06:46:53.902Z
189862024-02-08T06:46:53.902ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
189872024-02-08T06:46:53.921ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189882024-02-08T06:46:53.921Z 0 000-019 170 170 170 289 289 289 F F F
189892024-02-08T06:46:53.921Z 1 020-039 170 170 170 289 289 289 T T F <---
189902024-02-08T06:46:53.921Z 2 040-059 172 172 172 295 295 295 T T F <---
189912024-02-08T06:46:53.921Z 3 060-079 167 167 167 284 284 284 T T F <---
189922024-02-08T06:46:53.921Z 4 080-099 169 169 169 288 288 288 T T F <---
189932024-02-08T06:46:53.921Z 5 100-119 172 172 172 295 295 295 F F F
189942024-02-08T06:46:53.922Z 6 120-139 172 172 172 295 295 295 F F F
189952024-02-08T06:46:53.922Z 7 140-159 173 173 173 296 296 296 T T F <---
189962024-02-08T06:46:53.922Z 8 160-179 162 162 162 277 277 277 F F F
189972024-02-08T06:46:53.925Z 9 180-199 163 163 163 278 278 278 F F F
189982024-02-08T06:46:53.925Z 10 200-219 172 172 172 295 295 295 T T F <---
189992024-02-08T06:46:53.925Z 11 220-239 170 170 170 289 289 289 T T F <---
190002024-02-08T06:46:53.925Z 12 240-259 171 171 171 293 293 293 T T F <---
190012024-02-08T06:46:53.925Z 13 260-279 169 169 169 288 288 288 F F F
190022024-02-08T06:46:53.925Z 14 280-299 172 172 172 295 295 295 F F F
190032024-02-08T06:46:53.925Z 15 300-319 172 172 172 295 295 295 F F F
190042024-02-08T06:46:53.925Z 16 320-339 172 172 172 295 295 295 T T F <---
190052024-02-08T06:46:53.926Z 17 340-359 171 171 171 293 293 293 F F F
190062024-02-08T06:46:53.926Z 18 360-379 171 171 171 293 293 293 F F F
190072024-02-08T06:46:53.926Z 19 380-399 172 172 172 295 295 295 F F F
190082024-02-08T06:46:53.926Z 20 400-419 164 164 164 281 281 281 F F F
190092024-02-08T06:46:53.926Z 21 420-439 172 172 172 295 295 295 T T F <---
190102024-02-08T06:46:53.926Z 22 440-459 172 172 172 295 295 295 F F F
190112024-02-08T06:46:53.926Z 23 460-479 169 169 169 288 288 288 F F F
190122024-02-08T06:46:53.926Z 24 480-499 173 173 173 296 296 296 T T F <---
190132024-02-08T06:46:53.926Z 25 500-519 171 171 171 293 293 293 F F F
190142024-02-08T06:46:53.926Z 26 520-539 172 172 172 295 295 295 F F F
190152024-02-08T06:46:53.926Z 27 540-559 172 172 172 295 295 295 T T F <---
190162024-02-08T06:46:53.926Z 28 560-579 169 169 169 288 288 288 F F F
190172024-02-08T06:46:53.926Z 29 580-599 166 166 166 283 283 283 T T F <---
190182024-02-08T06:46:53.926ZMax gen: 173, Max flush: 296
190192024-02-08T06:46:53.926ZError: Difference in extent metadata found!
190202024-02-08T06:46:53.926ZA Difference in extent metadata is expected here
190212024-02-08T06:46:53.927ZOn loop 87
190222024-02-08T06:46:53.927Z
190232024-02-08T06:46:53.930ZVerifying data now
190242024-02-08T06:46:58.596ZNo change after verify
190252024-02-08T06:46:58.596ZLoop: 87 Downstairs dump after verify (and repair):
190262024-02-08T06:46:58.611ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190272024-02-08T06:46:58.611Z 0 000-019 170 170 170 289 289 289 F F F
190282024-02-08T06:46:58.611Z 1 020-039 174 174 174 297 297 297 F F F
190292024-02-08T06:46:58.611Z 2 040-059 174 174 174 297 297 297 F F F
190302024-02-08T06:46:58.611Z 3 060-079 174 174 174 297 297 297 F F F
190312024-02-08T06:46:58.612Z 4 080-099 174 174 174 297 297 297 F F F
190322024-02-08T06:46:58.612Z 5 100-119 172 172 172 295 295 295 F F F
190332024-02-08T06:46:58.612Z 6 120-139 172 172 172 295 295 295 F F F
190342024-02-08T06:46:58.612Z 7 140-159 174 174 174 297 297 297 F F F
190352024-02-08T06:46:58.612Z 8 160-179 162 162 162 277 277 277 F F F
190362024-02-08T06:46:58.612Z 9 180-199 163 163 163 278 278 278 F F F
190372024-02-08T06:46:58.612Z 10 200-219 174 174 174 297 297 297 F F F
190382024-02-08T06:46:58.615Z 11 220-239 174 174 174 297 297 297 F F F
190392024-02-08T06:46:58.615Z 12 240-259 174 174 174 297 297 297 F F F
190402024-02-08T06:46:58.615Z 13 260-279 169 169 169 288 288 288 F F F
190412024-02-08T06:46:58.616Z 14 280-299 172 172 172 295 295 295 F F F
190422024-02-08T06:46:58.616Z 15 300-319 172 172 172 295 295 295 F F F
190432024-02-08T06:46:58.616Z 16 320-339 174 174 174 297 297 297 F F F
190442024-02-08T06:46:58.616Z 17 340-359 171 171 171 293 293 293 F F F
190452024-02-08T06:46:58.616Z 18 360-379 171 171 171 293 293 293 F F F
190462024-02-08T06:46:58.616Z 19 380-399 172 172 172 295 295 295 F F F
190472024-02-08T06:46:58.616Z 20 400-419 164 164 164 281 281 281 F F F
190482024-02-08T06:46:58.616Z 21 420-439 174 174 174 297 297 297 F F F
190492024-02-08T06:46:58.616Z 22 440-459 172 172 172 295 295 295 F F F
190502024-02-08T06:46:58.616Z 23 460-479 169 169 169 288 288 288 F F F
190512024-02-08T06:46:58.616Z 24 480-499 174 174 174 297 297 297 F F F
190522024-02-08T06:46:58.616Z 25 500-519 171 171 171 293 293 293 F F F
190532024-02-08T06:46:58.616Z 26 520-539 172 172 172 295 295 295 F F F
190542024-02-08T06:46:58.616Z 27 540-559 174 174 174 297 297 297 F F F
190552024-02-08T06:46:58.616Z 28 560-579 169 169 169 288 288 288 F F F
190562024-02-08T06:46:58.617Z 29 580-599 174 174 174 297 297 297 F F F
190572024-02-08T06:46:58.617ZMax gen: 174, Max flush: 297
190582024-02-08T06:46:58.617Z
190592024-02-08T06:46:58.617ZBegin loop 88
190602024-02-08T06:46:58.617ZDownstairs to restart: 0
190612024-02-08T06:46:58.632Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844019455Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842}
190622024-02-08T06:46:58.632Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844309764Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842}
190632024-02-08T06:46:58.632Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.84434931Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842}
190642024-02-08T06:46:58.639Z{"msg":"Crucible cd10b465-71cd-4d5e-8be1-86e48a76ad60 has session id: 543ef580-999b-4511-9f07-e49230772859","v":0,"name":"crucible","level":30Crucible runtime is spawned
190652024-02-08T06:46:58.639Z,"time":"2024-02-08T06:46:57.844375999Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
190662024-02-08T06:46:58.640Z{"msg":"Upstairs opts: Upstairs UUID: cd10b465-71cd-4d5e-8be1-86e48a76ad60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844428995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
190672024-02-08T06:46:58.640Z{"msg":"Crucible stats registered with UUID: cd10b465-71cd-4d5e-8be1-86e48a76ad60","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844459402Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
190682024-02-08T06:46:58.640Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844482331Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842}
190692024-02-08T06:46:58.640Z{"msg":"cd10b465-71cd-4d5e-8be1-86e48a76ad60 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.84462211Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
190702024-02-08T06:46:58.640Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844679396Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190712024-02-08T06:46:58.640Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844716493Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190722024-02-08T06:46:58.640Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.84475146Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190732024-02-08T06:46:58.640Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844775869Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190742024-02-08T06:46:58.640Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844805316Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190752024-02-08T06:46:58.640Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844820805Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190762024-02-08T06:46:58.640Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844852783Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"io task","client":"0","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190772024-02-08T06:46:58.640Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844876941Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"io task","client":"1","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190782024-02-08T06:46:58.640Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844914088Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"io task","client":"2","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190792024-02-08T06:46:58.640Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.844961215Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"io task","client":"0","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190802024-02-08T06:46:58.640Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.845111923Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"io task","client":"2","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190812024-02-08T06:46:58.641Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.845171119Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"io task","client":"1","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190822024-02-08T06:46:58.641Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.845585548Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190832024-02-08T06:46:58.641Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.845637194Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190842024-02-08T06:46:58.641Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.845664292Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190852024-02-08T06:46:58.641Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.845713588Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190862024-02-08T06:46:58.641Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.846297474Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190872024-02-08T06:46:58.641Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.846336011Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190882024-02-08T06:46:58.641Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.846904879Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190892024-02-08T06:46:58.641Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.846962514Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190902024-02-08T06:46:58.641Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.846997482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
190912024-02-08T06:46:58.641Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.84701209Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190922024-02-08T06:46:58.641Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.847032879Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190932024-02-08T06:46:58.641Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.847047408Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
190942024-02-08T06:46:58.641Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.847228984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190952024-02-08T06:46:58.642Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.84728285Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190962024-02-08T06:46:58.646Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.847760084Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190972024-02-08T06:46:58.646Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.84821174Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190982024-02-08T06:46:58.647Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848257977Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
190992024-02-08T06:46:58.647Z{"msg":"[0]R flush_numbers[0..12]: [289, 297, 297, 297, 297, 295, 295, 297, 277, 278, 297, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848295554Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191002024-02-08T06:46:58.647Z{"msg":"[0]R generation[0..12]: [170, 174, 174, 174, 174, 172, 172, 174, 162, 163, 174, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848333301Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191012024-02-08T06:46:58.647Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848368319Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191022024-02-08T06:46:58.647Z{"msg":"[1]R flush_numbers[0..12]: [289, 297, 297, 297, 297, 295, 295, 297, 277, 278, 297, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848405236Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191032024-02-08T06:46:58.647Z{"msg":"[1]R generation[0..12]: [170, 174, 174, 174, 174, 172, 172, 174, 162, 163, 174, 174]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
191042024-02-08T06:46:58.647Z,"time":"2024-02-08T06:46:57.848443013Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191052024-02-08T06:46:58.647Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.84848454Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191062024-02-08T06:46:58.647Z{"msg":"[2]R flush_numbers[0..12]: [289, 297, 297, 297, 297, 295, 295, 297, 277, 278, 297, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848523687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191072024-02-08T06:46:58.647Z{"msg":"[2]R generation[0..12]: [170, 174, 174, 174, 174, 172, 172, 174, 162, 163, 174, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848563454Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191082024-02-08T06:46:58.647Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848602211Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191092024-02-08T06:46:58.647Z{"msg":"Max found gen is 175","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848633429Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191102024-02-08T06:46:58.647Z{"msg":"Generation requested: 176 >= found:175","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848648118Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191112024-02-08T06:46:58.647Z{"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848669856Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191122024-02-08T06:46:58.647Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848695464Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191132024-02-08T06:46:58.648Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848727202Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
191142024-02-08T06:46:58.648Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.84874639Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191152024-02-08T06:46:58.648Z{"msg":"Transition from WaitQuorum to ActiveRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191162024-02-08T06:46:58.648Z","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848778678Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191172024-02-08T06:46:58.648Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848799816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"543ef580-999b-4511-9f07-e49230772859"}
191182024-02-08T06:46:58.648Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848827054Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
191192024-02-08T06:46:58.648Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848843763Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
191202024-02-08T06:46:58.648Z{"msg":"cd10b465-71cd-4d5e-8be1-86e48a76ad60 is now active with session: 543ef580-999b-4511-9f07-e49230772859","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848872981Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
191212024-02-08T06:46:58.648Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.84888996Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842,"session_id":"543ef580-999b-4511-9f07-e49230772859"}
191222024-02-08T06:46:58.648Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:46:57.848917698Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842}
191232024-02-08T06:46:58.648ZLoading write count information from file "/tmp/test_repair_verify.data"
191242024-02-08T06:46:58.648ZRun Repair workload
191252024-02-08T06:46:58.648Z01/30 Write block 550 len 24576 data: 28 29 28 25 22 20
191262024-02-08T06:46:58.648Z02/30 Write block 314 len 36864 data: 11 7 8 8 10 12 17 15 17
191272024-02-08T06:46:58.648Z03/30 Write block 156 len 12288 data: 16 14 15
191282024-02-08T06:46:58.649Z04/30 Flush
191292024-02-08T06:46:58.649Z05/30 Write block 351 len 32768 data: 13 13 12 12 12 10 12 12
191302024-02-08T06:46:58.649Z06/30 Write block 309 len 20480 data: 9 9 9 10 9
191312024-02-08T06:46:58.652Z07/30 Write block 332 len 20480 data: 12 15 18 18 20
191322024-02-08T06:46:58.652Z08/30 Flush
191332024-02-08T06:46:58.652Z09/30 Write block 238 len 20480 data: 17 18 15 19 15
191342024-02-08T06:46:58.652Z10/30 Write block 76 len 12288 data: 13 11 11
191352024-02-08T06:46:58.652Z11/30 Read block 538 len 8192
191362024-02-08T06:46:58.652Z12/30 Write block 407 len 16384 data: 13 13 13 11
191372024-02-08T06:46:58.652Z13/30 Write block 92 len 36864 data: 17 16 19 20 20 21 19 19 20
191382024-02-08T06:46:58.652Z14/30 Write block 330 len 24576 data: 7 11 13 16 19 19
191392024-02-08T06:46:58.655Z15/30 Write block 330 len 12288 data: 8 12 14
191402024-02-08T06:46:58.655Z16/30 Read block 402 len 28672
191412024-02-08T06:46:58.655Z17/30 Read block 146 len 32768
191422024-02-08T06:46:58.656Z18/30 Read block 418 len 36864
191432024-02-08T06:46:58.659Z19/30 Write block 381 len 8192 data: 9 12
191442024-02-08T06:46:58.659Z20/30 Write block 13 len 36864 data: 13 12 12 11 11 12 13 15 12
191452024-02-08T06:46:58.659Z21/30 Write block 518 len 4096 data: 11
191462024-02-08T06:46:58.659Z22/30 Read block 386 len 24576
191472024-02-08T06:46:58.659Z23/30 Read block 179 len 16384
191482024-02-08T06:46:58.662Z24/30 Read block 240 len 36864
191492024-02-08T06:46:58.662Z25/30 Write block 257 len 28672 data: 10 11 14 14 20 20 16
191502024-02-08T06:46:58.662Z26/30 Write block 96 len 20480 data: 21 22 20 20 21
191512024-02-08T06:46:58.662Z27/30 Read block 37 len 20480
191522024-02-08T06:46:58.666Z28/30 Write block 339 len 8192 data: 20 18
191532024-02-08T06:46:58.666Z29/30 Write block 329 len 4096 data: 7
191542024-02-08T06:46:58.666Z30/30 Write block 539 len 40960 data: 13 13 12 10 13 13 14 18 23 27
191552024-02-08T06:46:58.666Z----------------------------------------------------------------
191562024-02-08T06:46:58.666Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
191572024-02-08T06:46:58.666ZStates: Active Active Active
191582024-02-08T06:46:58.666ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
191592024-02-08T06:46:58.666Z 1 Acked 1000 Write 6 Done Done Done false
191602024-02-08T06:46:58.666Z 2 Acked 1001 Write 9 Sent Done Done false
191612024-02-08T06:46:58.667Z 3 Acked 1002 Write 3 Sent Done Done false
191622024-02-08T06:46:58.667Z 4 Acked 1003 Flush 0 Sent Done Done false
191632024-02-08T06:46:58.667Z 5 Acked 1004 Write 8 Sent Done Done false
191642024-02-08T06:46:58.667Z 6 Acked 1005 Write 5 Sent Done Done false
191652024-02-08T06:46:58.667Z 7 Acked 1006 Write 5 Sent Done Done false
191662024-02-08T06:46:58.667Z 8 Acked 1007 Flush 0 Sent Done Done false
191672024-02-08T06:46:58.667Z 9 Acked 1008 Write 5 Sent Done Done false
191682024-02-08T06:46:58.667Z 10 Acked 1009 Write 3 Sent Done Done false
191692024-02-08T06:46:58.667Z 11 Acked 1010 Read 2 Sent Done Done false
191702024-02-08T06:46:58.667Z 12 Acked 1011 Write 4 Sent Done Done false
191712024-02-08T06:46:58.667Z 13 Acked 1012 Write 9 Sent Done Done false
191722024-02-08T06:46:58.667Z 14 Acked 1013 Write 6 Sent Done Done false
191732024-02-08T06:46:58.667Z 15 Acked 1014 Write 3 Sent Done Done false
191742024-02-08T06:46:58.667Z 16 Acked 1015 Read 7 Sent Done Done false
191752024-02-08T06:46:58.667Z 17 Acked 1016 Read 8 Sent Done Done false
191762024-02-08T06:46:58.668Z 18 Acked 1017 Read 9 Sent Done Done false
191772024-02-08T06:46:58.668Z 19 Acked 1018 Write 2 Sent Done Done false
191782024-02-08T06:46:58.668Z 20 Acked 1019 Write 9 Sent Done Done false
191792024-02-08T06:46:58.668Z 21 Acked 1020 Write 1 Sent Done Done false
191802024-02-08T06:46:58.668Z 22 Acked 1021 Read 6 Sent Done Done false
191812024-02-08T06:46:58.668Z 23 Acked 1022 Read 4 Sent Done Done false
191822024-02-08T06:46:58.668Z 24 Acked 1023 Read 9 Sent Done Done false
191832024-02-08T06:46:58.668Z 25 Acked 1024 Write 7 Sent Done Done false
191842024-02-08T06:46:58.668Z 26 Acked 1025 Write 5 Sent Done Done false
191852024-02-08T06:46:58.668Z 27 Acked 1026 Read 5 Sent Done Done false
191862024-02-08T06:46:58.668Z 28 Acked 1027 Write 2 Sent Done Done false
191872024-02-08T06:46:58.668Z 29 Acked 1028 Write 1 Sent Done Done false
191882024-02-08T06:46:58.668Z 30 Acked 1029 Write 10 Sent Sent Sent false
191892024-02-08T06:46:58.668Z STATES DS:0 DS:1 DS:2 TOTAL
191902024-02-08T06:46:58.668Z New 0 0 0 0
191912024-02-08T06:46:58.669Z Sent 29 1 1 31
191922024-02-08T06:46:58.669Z Done 1 29 29 59
191932024-02-08T06:46:58.669Z Skipped 0 0 0 0
191942024-02-08T06:46:58.669Z Error 0 0 0 0
191952024-02-08T06:46:58.669ZLast Flush: 0 1007 1007
191962024-02-08T06:46:58.669ZDownstairs last five completed:
191972024-02-08T06:46:58.669ZUpstairs last five completed: 30 29 28 27 26
191982024-02-08T06:46:58.669Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:46:57.877176206Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1842}
191992024-02-08T06:46:58.672ZWrote out file "/tmp/test_repair_verify.data"
192002024-02-08T06:46:58.672Z
192012024-02-08T06:46:58.673ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192022024-02-08T06:46:58.693ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192032024-02-08T06:46:58.693Z 0 000-019 170 170 170 289 289 289 F T T <---
192042024-02-08T06:46:58.693Z 1 020-039 174 174 174 297 297 297 F T T <---
192052024-02-08T06:46:58.693Z 2 040-059 174 174 174 297 297 297 F F F
192062024-02-08T06:46:58.693Z 3 060-079 174 174 174 297 297 297 F T T <---
192072024-02-08T06:46:58.693Z 4 080-099 174 174 174 297 297 297 F T T <---
192082024-02-08T06:46:58.693Z 5 100-119 172 172 172 295 295 295 F T T <---
192092024-02-08T06:46:58.693Z 6 120-139 172 172 172 295 295 295 F F F
192102024-02-08T06:46:58.696Z 7 140-159 174 176 176 297 298 298 F F F <---
192112024-02-08T06:46:58.696Z 8 160-179 162 162 162 277 277 277 F F F
192122024-02-08T06:46:58.696Z 9 180-199 163 163 163 278 278 278 F F F
192132024-02-08T06:46:58.696Z 10 200-219 174 174 174 297 297 297 F F F
192142024-02-08T06:46:58.696Z 11 220-239 174 174 174 297 297 297 F T T <---
192152024-02-08T06:46:58.697Z 12 240-259 174 174 174 297 297 297 F T T <---
192162024-02-08T06:46:58.697Z 13 260-279 169 169 169 288 288 288 F T T <---
192172024-02-08T06:46:58.697Z 14 280-299 172 172 172 295 295 295 F F F
192182024-02-08T06:46:58.697Z 15 300-319 172 176 176 295 299 299 F F F <---
192192024-02-08T06:46:58.697Z 16 320-339 174 176 176 297 299 299 F T T <---
192202024-02-08T06:46:58.697Z 17 340-359 171 176 176 293 299 299 F T T <---
192212024-02-08T06:46:58.697Z 18 360-379 171 171 171 293 293 293 F F F
192222024-02-08T06:46:58.697Z 19 380-399 172 172 172 295 295 295 F T T <---
192232024-02-08T06:46:58.697Z 20 400-419 164 164 164 281 281 281 F T T <---
192242024-02-08T06:46:58.697Z 21 420-439 174 174 174 297 297 297 F F F
192252024-02-08T06:46:58.697Z 22 440-459 172 172 172 295 295 295 F F F
192262024-02-08T06:46:58.697Z 23 460-479 169 169 169 288 288 288 F F F
192272024-02-08T06:46:58.697Z 24 480-499 174 174 174 297 297 297 F F F
192282024-02-08T06:46:58.697Z 25 500-519 171 171 171 293 293 293 F T T <---
192292024-02-08T06:46:58.697Z 26 520-539 172 172 172 295 295 295 F T T <---
192302024-02-08T06:46:58.697Z 27 540-559 174 176 176 297 298 298 T T T <---
192312024-02-08T06:46:58.698Z 28 560-579 169 169 169 288 288 288 F F F
192322024-02-08T06:46:58.698Z 29 580-599 174 174 174 297 297 297 F F F
192332024-02-08T06:46:58.698ZMax gen: 176, Max flush: 299
192342024-02-08T06:46:58.698ZError: Difference in extent metadata found!
192352024-02-08T06:46:58.698ZA Difference in extent metadata is expected here
192362024-02-08T06:46:58.698ZOn loop 88
192372024-02-08T06:46:58.698Z
192382024-02-08T06:46:58.701ZVerifying data now
192392024-02-08T06:47:04.526ZNo change after verify
192402024-02-08T06:47:04.526ZLoop: 88 Downstairs dump after verify (and repair):
192412024-02-08T06:47:04.540ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192422024-02-08T06:47:04.540Z 0 000-019 177 177 177 300 300 300 F F F
192432024-02-08T06:47:04.540Z 1 020-039 177 177 177 300 300 300 F F F
192442024-02-08T06:47:04.540Z 2 040-059 174 174 174 297 297 297 F F F
192452024-02-08T06:47:04.540Z 3 060-079 177 177 177 300 300 300 F F F
192462024-02-08T06:47:04.540Z 4 080-099 177 177 177 300 300 300 F F F
192472024-02-08T06:47:04.540Z 5 100-119 177 177 177 300 300 300 F F F
192482024-02-08T06:47:04.541Z 6 120-139 172 172 172 295 295 295 F F F
192492024-02-08T06:47:04.544Z 7 140-159 176 176 176 298 298 298 F F F
192502024-02-08T06:47:04.544Z 8 160-179 162 162 162 277 277 277 F F F
192512024-02-08T06:47:04.545Z 9 180-199 163 163 163 278 278 278 F F F
192522024-02-08T06:47:04.545Z 10 200-219 174 174 174 297 297 297 F F F
192532024-02-08T06:47:04.545Z 11 220-239 177 177 177 300 300 300 F F F
192542024-02-08T06:47:04.545Z 12 240-259 177 177 177 300 300 300 F F F
192552024-02-08T06:47:04.545Z 13 260-279 177 177 177 300 300 300 F F F
192562024-02-08T06:47:04.545Z 14 280-299 172 172 172 295 295 295 F F F
192572024-02-08T06:47:04.545Z 15 300-319 176 176 176 299 299 299 F F F
192582024-02-08T06:47:04.545Z 16 320-339 177 177 177 300 300 300 F F F
192592024-02-08T06:47:04.545Z 17 340-359 177 177 177 300 300 300 F F F
192602024-02-08T06:47:04.545Z 18 360-379 171 171 171 293 293 293 F F F
192612024-02-08T06:47:04.545Z 19 380-399 177 177 177 300 300 300 F F F
192622024-02-08T06:47:04.545Z 20 400-419 177 177 177 300 300 300 F F F
192632024-02-08T06:47:04.545Z 21 420-439 174 174 174 297 297 297 F F F
192642024-02-08T06:47:04.545Z 22 440-459 172 172 172 295 295 295 F F F
192652024-02-08T06:47:04.546Z 23 460-479 169 169 169 288 288 288 F F F
192662024-02-08T06:47:04.546Z 24 480-499 174 174 174 297 297 297 F F F
192672024-02-08T06:47:04.546Z 25 500-519 177 177 177 300 300 300 F F F
192682024-02-08T06:47:04.546Z 26 520-539 177 177 177 300 300 300 F F F
192692024-02-08T06:47:04.546Z 27 540-559 177 177 177 300 300 300 F F F
192702024-02-08T06:47:04.546Z 28 560-579 169 169 169 288 288 288 F F F
192712024-02-08T06:47:04.546Z 29 580-599 174 174 174 297 297 297 F F F
192722024-02-08T06:47:04.546ZMax gen: 177, Max flush: 300
192732024-02-08T06:47:04.546Z
192742024-02-08T06:47:04.546ZBegin loop 89
192752024-02-08T06:47:04.546ZDownstairs to restart: 2
192762024-02-08T06:47:04.561Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771231454Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850}
192772024-02-08T06:47:04.562Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
192782024-02-08T06:47:04.562Z2024-02-08T06:47:03.771542301Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850}
192792024-02-08T06:47:04.568Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771585138Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850}
192802024-02-08T06:47:04.568Z{"msg":"Crucible 18035783-88b8-459b-ae23-65e91db15933 has session id: f57ad680-f847-4d2e-b188-ee34c7b621a5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771626534Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192812024-02-08T06:47:04.568Z{"msg":"Upstairs opts: Upstairs UUID: 18035783-88b8-459b-ae23-65e91db15933, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771665431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192822024-02-08T06:47:04.568Z{"msg":"Crucible stats registered with UUID: 18035783-88b8-459b-ae23-65e91db15933","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771698889Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192832024-02-08T06:47:04.568Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771727227Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850}
192842024-02-08T06:47:04.568Z{"msg":"18035783-88b8-459b-ae23-65e91db15933 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771758445Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192852024-02-08T06:47:04.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771786402Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192862024-02-08T06:47:04.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771833729Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192872024-02-08T06:47:04.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771867256Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192882024-02-08T06:47:04.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771898624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192892024-02-08T06:47:04.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.771926612Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192902024-02-08T06:47:04.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.77195438Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192912024-02-08T06:47:04.569Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.772042403Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"io task","client":"0","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192922024-02-08T06:47:04.569Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.77208117Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"io task","client":"1","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192932024-02-08T06:47:04.569Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.772108328Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"io task","client":"2","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192942024-02-08T06:47:04.569Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.772134626Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"io task","client":"0","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192952024-02-08T06:47:04.569Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.772176833Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"io task","client":"1","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192962024-02-08T06:47:04.569Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.772378888Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"io task","client":"2","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192972024-02-08T06:47:04.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.772959744Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192982024-02-08T06:47:04.569Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.77300958Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
192992024-02-08T06:47:04.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.773108263Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193002024-02-08T06:47:04.570Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.77315252Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193012024-02-08T06:47:04.570Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.773589157Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193022024-02-08T06:47:04.570Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.773647683Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193032024-02-08T06:47:04.570Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.774199331Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193042024-02-08T06:47:04.570Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.774259927Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193052024-02-08T06:47:04.570Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.774296244Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193062024-02-08T06:47:04.570Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.774311213Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193072024-02-08T06:47:04.570Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.774325502Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193082024-02-08T06:47:04.570Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.77434755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193092024-02-08T06:47:04.570Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.774361149Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193102024-02-08T06:47:04.570Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.774393367Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193112024-02-08T06:47:04.570Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.774926327Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193122024-02-08T06:47:04.570Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775363184Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193132024-02-08T06:47:04.570Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775396011Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193142024-02-08T06:47:04.570Z{"msg":"[0]R flush_numbers[0..12]: [300, 300, 297, 300, 300, 300, 295, 298, 277, 278, 297, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.77541137Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193152024-02-08T06:47:04.571Z{"msg":"[0]R generation[0..12]: [177, 177, 174, 177, 177, 177, 172, 176, 162, 163, 174, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775426719Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193162024-02-08T06:47:04.571Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775448357Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193172024-02-08T06:47:04.571Z{"msg":"[1]R flush_numbers[0..12]: [300, 300, 297, 300, 300, 300, 295, 298, 277, 278, 297, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775473515Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193182024-02-08T06:47:04.571Z{"msg":"[1]R generation[0..12]: [177, 177, 174, 177, 177, 177, 172, 176, 162, 163, 174, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775496004Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193192024-02-08T06:47:04.571Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775510723Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193202024-02-08T06:47:04.571Z{"msg":"[2]R flush_numbers[0..12]: [300, 300, 297, 300, 300, 300, 295, 298, 277, 278, 297, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775531991Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193212024-02-08T06:47:04.571Z{"msg":"[2]R generation[0..12]: [177, 177, 174, 177, 177, 177, 172, 176, 162, 163, 174, 177]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
193222024-02-08T06:47:04.571Z,"time":"2024-02-08T06:47:03.77554616Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193232024-02-08T06:47:04.571Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775560969Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193242024-02-08T06:47:04.571Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775581857Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193252024-02-08T06:47:04.571Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775595576Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193262024-02-08T06:47:04.571Z{"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775608505Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193272024-02-08T06:47:04.571Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775628354Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193282024-02-08T06:47:04.571Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775642003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193292024-02-08T06:47:04.571Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775669441Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193302024-02-08T06:47:04.572Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775694049Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193312024-02-08T06:47:04.572Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775726727Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193322024-02-08T06:47:04.572Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775742355Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193332024-02-08T06:47:04.572Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775766744Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193342024-02-08T06:47:04.572Z{"msg":"18035783-88b8-459b-ae23-65e91db15933 is now active with session: f57ad680-f847-4d2e-b188-ee34c7b621a5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775800881Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193352024-02-08T06:47:04.572Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775826149Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850,"session_id":"f57ad680-f847-4d2e-b188-ee34c7b621a5"}
193362024-02-08T06:47:04.572Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:03.775857587Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850}
193372024-02-08T06:47:04.572ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193382024-02-08T06:47:04.572ZLoading write count information from file "/tmp/test_repair_verify.data"
193392024-02-08T06:47:04.572ZRun Repair workload
193402024-02-08T06:47:04.576Z01/30 Write block 536 len 24576 data: 15 14 14 14 14 13
193412024-02-08T06:47:04.576Z02/30 Write block 234 len 8192 data: 11 15
193422024-02-08T06:47:04.576Z03/30 Write block 496 len 12288 data: 10 8 10
193432024-02-08T06:47:04.576Z04/30 Write block 133 len 20480 data: 11 10 10 11 15
193442024-02-08T06:47:04.576Z05/30 Write block 308 len 36864 data: 9 10 10 10 11 10 12 8 9
193452024-02-08T06:47:04.576Z06/30 Read block 2 len 16384
193462024-02-08T06:47:04.576Z07/30 Write block 83 len 40960 data: 15 15 15 14 13 12 11 12 15 18
193472024-02-08T06:47:04.576Z08/30 Write block 491 len 40960 data: 15 15 15 14 11 11 9 11 11 12
193482024-02-08T06:47:04.576Z09/30 Write block 418 len 40960 data: 16 16 15 15 13 13 12 10 10 7
193492024-02-08T06:47:04.577Z10/30 Read block 448 len 12288
193502024-02-08T06:47:04.577Z11/30 Read block 557 len 24576
193512024-02-08T06:47:04.577Z12/30 Read block 278 len 32768
193522024-02-08T06:47:04.582Z13/30 Flush
193532024-02-08T06:47:04.588Z14/30 Write block 285 len 4096 data: 12
193542024-02-08T06:47:04.588Z15/30 Flush
193552024-02-08T06:47:04.588Z16/30 Write block 358 len 12288 data: 13 15 14
193562024-02-08T06:47:04.588Z17/30 Read block 20 len 4096
193572024-02-08T06:47:04.588Z18/30 Write block 532 len 4096 data: 16
193582024-02-08T06:47:04.588Z19/30 Flush
193592024-02-08T06:47:04.616Z20/30 Write block 188 len 16384 data: 13 12 9 10
193602024-02-08T06:47:04.616Z21/30 Read block 392 len 4096
193612024-02-08T06:47:04.617Z22/30 Write block 190 len 20480 data: 10 11 9 11 13
193622024-02-08T06:47:04.617Z23/30 Write block 65 len 8192 data: 9 8
193632024-02-08T06:47:04.617Z24/30 Write block 408 len 28672 data: 14 14 12 10 11 10 11
193642024-02-08T06:47:04.617Z25/30 Write block 436 len 16384 data: 14 17 16 13
193652024-02-08T06:47:04.617Z26/30 Write block 385 len 4096 data: 12
193662024-02-08T06:47:04.617Z27/30 Write block 275 len 40960 data: 14 16 16 14 18 16 14 12 11 13
193672024-02-08T06:47:04.618Z28/30 Write block 479 len 40960 data: 9 7 8 9 8 13 14 16 15 18
193682024-02-08T06:47:04.618Z29/30 Write block 57 len 8192 data: 16 11
193692024-02-08T06:47:04.618Z30/30 Write block 324 len 16384 data: 14 17 15 11
193702024-02-08T06:47:04.618Z----------------------------------------------------------------
193712024-02-08T06:47:04.618Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
193722024-02-08T06:47:04.618ZStates: Active Active Active
193732024-02-08T06:47:04.618ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
193742024-02-08T06:47:04.619Z 1 Acked 1000 Write 6 Done Done Done false
193752024-02-08T06:47:04.619Z 2 Acked 1001 Write 2 Done Done Done false
193762024-02-08T06:47:04.619Z 3 Acked 1002 Write 3 Done Done Done false
193772024-02-08T06:47:04.619Z 4 Acked 1003 Write 5 Done Done Done false
193782024-02-08T06:47:04.619Z 5 Acked 1004 Write 9 Done Done Done false
193792024-02-08T06:47:04.619Z 6 Acked 1005 Read 4 Done Done Done false
193802024-02-08T06:47:04.620Z 7 Acked 1006 Write 10 Done Done Done false
193812024-02-08T06:47:04.620Z 8 Acked 1007 Write 10 Done Done Sent false
193822024-02-08T06:47:04.620Z 9 Acked 1008 Write 10 Done Done Sent false
193832024-02-08T06:47:04.620Z 10 Acked 1009 Read 3 Done Done Sent false
193842024-02-08T06:47:04.620Z 11 Acked 1010 Read 6 Done Done Sent false
193852024-02-08T06:47:04.620Z 12 Acked 1011 Read 8 Done Done Sent false
193862024-02-08T06:47:04.620Z 13 Acked 1012 Flush 0 Done Done Sent false
193872024-02-08T06:47:04.620Z 14 Acked 1013 Write 1 Done Done Sent false
193882024-02-08T06:47:04.620Z 15 Acked 1014 Flush 0 Done Done Sent false
193892024-02-08T06:47:04.620Z 16 Acked 1015 Write 3 Done Done Sent false
193902024-02-08T06:47:04.620Z 17 Acked 1016 Read 1 Done Done Sent false
193912024-02-08T06:47:04.620Z 18 Acked 1017 Write 1 Done Done Sent false
193922024-02-08T06:47:04.620Z 19 Acked 1018 Flush 0 Done Done Sent false
193932024-02-08T06:47:04.620Z 20 Acked 1019 Write 4 Done Done Sent false
193942024-02-08T06:47:04.621Z 21 Acked 1020 Read 1 Done Done Sent false
193952024-02-08T06:47:04.621Z 22 Acked 1021 Write 5 Done Done Sent false
193962024-02-08T06:47:04.621Z 23 Acked 1022 Write 2 Done Done Sent false
193972024-02-08T06:47:04.621Z 24 Acked 1023 Write 7 Done Done Sent false
193982024-02-08T06:47:04.621Z 25 Acked 1024 Write 4 Done Done Sent false
193992024-02-08T06:47:04.621Z 26 Acked 1025 Write 1 Done Done Sent false
194002024-02-08T06:47:04.621Z 27 Acked 1026 Write 10 Done Done Sent false
194012024-02-08T06:47:04.621Z 28 Acked 1027 Write 10 Done Done Sent false
194022024-02-08T06:47:04.621Z 29 Acked 1028 Write 2 Sent Sent Sent false
194032024-02-08T06:47:04.622Z 30 Acked 1029 Write 4 Sent Sent Sent false
194042024-02-08T06:47:04.622Z STATES DS:0 DS:1 DS:2 TOTAL
194052024-02-08T06:47:04.622Z New 0 0 0 0
194062024-02-08T06:47:04.622Z Sent 2 2 23 27
194072024-02-08T06:47:04.622Z Done 28 28 7 63
194082024-02-08T06:47:04.622Z Skipped 0 0 0 0
194092024-02-08T06:47:04.622Z Error 0 0 0 0
194102024-02-08T06:47:04.622ZLast Flush: 1018 1018 0
194112024-02-08T06:47:04.622ZDownstairs last five completed:
194122024-02-08T06:47:04.623ZUpstairs last five completed: 30 29 28 27 26
194132024-02-08T06:47:04.623Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:03.805852395Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1850}
194142024-02-08T06:47:04.623ZWrote out file "/tmp/test_repair_verify.data"
194152024-02-08T06:47:04.633Z
194162024-02-08T06:47:04.633ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194172024-02-08T06:47:04.633ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194182024-02-08T06:47:04.633Z 0 000-019 177 177 177 300 300 300 F F F
194192024-02-08T06:47:04.634Z 1 020-039 177 177 177 300 300 300 F F F
194202024-02-08T06:47:04.634Z 2 040-059 174 174 174 297 297 297 T T F <---
194212024-02-08T06:47:04.634Z 3 060-079 177 177 177 300 300 300 T T F <---
194222024-02-08T06:47:04.634Z 4 080-099 178 178 177 301 301 300 F F T <---
194232024-02-08T06:47:04.634Z 5 100-119 177 177 177 300 300 300 F F F
194242024-02-08T06:47:04.634Z 6 120-139 178 178 172 301 301 295 F F T <---
194252024-02-08T06:47:04.634Z 7 140-159 176 176 176 298 298 298 F F F
194262024-02-08T06:47:04.634Z 8 160-179 162 162 162 277 277 277 F F F
194272024-02-08T06:47:04.634Z 9 180-199 163 163 163 278 278 278 T T F <---
194282024-02-08T06:47:04.634Z 10 200-219 174 174 174 297 297 297 F F F
194292024-02-08T06:47:04.634Z 11 220-239 178 178 177 301 301 300 F F T <---
194302024-02-08T06:47:04.634Z 12 240-259 177 177 177 300 300 300 F F F
194312024-02-08T06:47:04.634Z 13 260-279 177 177 177 300 300 300 T T F <---
194322024-02-08T06:47:04.634Z 14 280-299 178 178 172 302 302 295 T T F <---
194332024-02-08T06:47:04.634Z 15 300-319 178 178 176 301 301 299 F F T <---
194342024-02-08T06:47:04.634Z 16 320-339 177 177 177 300 300 300 T T F <---
194352024-02-08T06:47:04.635Z 17 340-359 178 178 177 303 303 300 F F F <---
194362024-02-08T06:47:04.635Z 18 360-379 178 178 171 303 303 293 F F F <---
194372024-02-08T06:47:04.635Z 19 380-399 177 177 177 300 300 300 T T F <---
194382024-02-08T06:47:04.635Z 20 400-419 178 178 177 301 301 300 T T F <---
194392024-02-08T06:47:04.635Z 21 420-439 178 178 174 301 301 297 T T F <---
194402024-02-08T06:47:04.635Z 22 440-459 172 172 172 295 295 295 F F F
194412024-02-08T06:47:04.635Z 23 460-479 169 169 169 288 288 288 T T F <---
194422024-02-08T06:47:04.635Z 24 480-499 178 178 174 301 301 297 T T T <---
194432024-02-08T06:47:04.635Z 25 500-519 178 178 177 301 301 300 F F F <---
194442024-02-08T06:47:04.635Z 26 520-539 178 178 177 303 303 300 F F T <---
194452024-02-08T06:47:04.635Z 27 540-559 178 178 177 301 301 300 F F T <---
194462024-02-08T06:47:04.635Z 28 560-579 169 169 169 288 288 288 F F F
194472024-02-08T06:47:04.635Z 29 580-599 174 174 174 297 297 297 F F F
194482024-02-08T06:47:04.635ZMax gen: 178, Max flush: 303
194492024-02-08T06:47:04.635ZError: Difference in extent metadata found!
194502024-02-08T06:47:04.636ZA Difference in extent metadata is expected here
194512024-02-08T06:47:04.636ZOn loop 89
194522024-02-08T06:47:04.636Z
194532024-02-08T06:47:04.636ZVerifying data now
194542024-02-08T06:47:10.577ZNo change after verify
194552024-02-08T06:47:10.580ZLoop: 89 Downstairs dump after verify (and repair):
194562024-02-08T06:47:10.591ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194572024-02-08T06:47:10.591Z 0 000-019 177 177 177 300 300 300 F F F
194582024-02-08T06:47:10.591Z 1 020-039 177 177 177 300 300 300 F F F
194592024-02-08T06:47:10.592Z 2 040-059 179 179 179 304 304 304 F F F
194602024-02-08T06:47:10.592Z 3 060-079 179 179 179 304 304 304 F F F
194612024-02-08T06:47:10.592Z 4 080-099 178 178 178 301 301 301 F F F
194622024-02-08T06:47:10.595Z 5 100-119 177 177 177 300 300 300 F F F
194632024-02-08T06:47:10.595Z 6 120-139 178 178 178 301 301 301 F F F
194642024-02-08T06:47:10.595Z 7 140-159 176 176 176 298 298 298 F F F
194652024-02-08T06:47:10.595Z 8 160-179 162 162 162 277 277 277 F F F
194662024-02-08T06:47:10.595Z 9 180-199 179 179 179 304 304 304 F F F
194672024-02-08T06:47:10.595Z 10 200-219 174 174 174 297 297 297 F F F
194682024-02-08T06:47:10.595Z 11 220-239 178 178 178 301 301 301 F F F
194692024-02-08T06:47:10.596Z 12 240-259 177 177 177 300 300 300 F F F
194702024-02-08T06:47:10.596Z 13 260-279 179 179 179 304 304 304 F F F
194712024-02-08T06:47:10.596Z 14 280-299 179 179 179 304 304 304 F F F
194722024-02-08T06:47:10.596Z 15 300-319 178 178 178 301 301 301 F F F
194732024-02-08T06:47:10.596Z 16 320-339 179 179 179 304 304 304 F F F
194742024-02-08T06:47:10.596Z 17 340-359 178 178 178 303 303 303 F F F
194752024-02-08T06:47:10.596Z 18 360-379 178 178 178 303 303 303 F F F
194762024-02-08T06:47:10.596Z 19 380-399 179 179 179 304 304 304 F F F
194772024-02-08T06:47:10.596Z 20 400-419 179 179 179 304 304 304 F F F
194782024-02-08T06:47:10.596Z 21 420-439 179 179 179 304 304 304 F F F
194792024-02-08T06:47:10.596Z 22 440-459 172 172 172 295 295 295 F F F
194802024-02-08T06:47:10.596Z 23 460-479 179 179 179 304 304 304 F F F
194812024-02-08T06:47:10.596Z 24 480-499 179 179 179 304 304 304 F F F
194822024-02-08T06:47:10.596Z 25 500-519 178 178 178 301 301 301 F F F
194832024-02-08T06:47:10.596Z 26 520-539 178 178 178 303 303 303 F F F
194842024-02-08T06:47:10.596Z 27 540-559 178 178 178 301 301 301 F F F
194852024-02-08T06:47:10.597Z 28 560-579 169 169 169 288 288 288 F F F
194862024-02-08T06:47:10.597Z 29 580-599 174 174 174 297 297 297 F F F
194872024-02-08T06:47:10.597ZMax gen: 179, Max flush: 304
194882024-02-08T06:47:10.597Z
194892024-02-08T06:47:10.597ZBegin loop 90
194902024-02-08T06:47:10.597ZDownstairs to restart: 2
194912024-02-08T06:47:10.612Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.822749929Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858}
194922024-02-08T06:47:10.612ZCrucible runtime is spawned
194932024-02-08T06:47:10.612Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823104372Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858}
194942024-02-08T06:47:10.613Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.82313522Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858}
194952024-02-08T06:47:10.618Z{"msg":"Crucible 64c5ac96-05bd-48a6-97c0-35b78366bf40 has session id: 88ac6fa7-60bf-41f6-b72b-3069329efe3b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823162358Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
194962024-02-08T06:47:10.618Z{"msg":"Upstairs opts: Upstairs UUID: 64c5ac96-05bd-48a6-97c0-35b78366bf40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823209674Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
194972024-02-08T06:47:10.618Z{"msg":"Crucible stats registered with UUID: 64c5ac96-05bd-48a6-97c0-35b78366bf40","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823236572Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
194982024-02-08T06:47:10.618Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.82326889Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858}
194992024-02-08T06:47:10.619Z{"msg":"64c5ac96-05bd-48a6-97c0-35b78366bf40 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823283289Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195002024-02-08T06:47:10.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823296108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195012024-02-08T06:47:10.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823338355Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195022024-02-08T06:47:10.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823366303Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195032024-02-08T06:47:10.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823392811Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195042024-02-08T06:47:10.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823417939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195052024-02-08T06:47:10.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823442767Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195062024-02-08T06:47:10.619Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823479834Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"io task","client":"0","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195072024-02-08T06:47:10.619Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823508382Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"io task","client":"1","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195082024-02-08T06:47:10.619Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823544799Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"io task","client":"2","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195092024-02-08T06:47:10.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823574967Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"io task","client":"1","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195102024-02-08T06:47:10.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823610574Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"io task","client":"2","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195112024-02-08T06:47:10.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.823657001Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"io task","client":"0","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195122024-02-08T06:47:10.619Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.82433625Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195132024-02-08T06:47:10.619Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.824364108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195142024-02-08T06:47:10.620Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.824379027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195152024-02-08T06:47:10.620Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.824411774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195162024-02-08T06:47:10.620Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.824918426Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195172024-02-08T06:47:10.620Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.824975722Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195182024-02-08T06:47:10.620Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.825331365Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195192024-02-08T06:47:10.620Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.825378122Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195202024-02-08T06:47:10.620Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.825408929Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195212024-02-08T06:47:10.620Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.825450446Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195222024-02-08T06:47:10.620Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.825476864Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195232024-02-08T06:47:10.620Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.825596765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195242024-02-08T06:47:10.620Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.825625003Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195252024-02-08T06:47:10.620Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.825640022Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195262024-02-08T06:47:10.620Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826040732Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195272024-02-08T06:47:10.620Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826500337Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195282024-02-08T06:47:10.620Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826549704Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195292024-02-08T06:47:10.620Z{"msg":"[0]R flush_numbers[0..12]: [300, 300, 304, 304, 301, 300, 301, 298, 277, 304, 297, 301]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826604209Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195302024-02-08T06:47:10.621Z{"msg":"[0]R generation[0..12]: [177, 177, 179, 179, 178, 177, 178, 176, 162, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826640817Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195312024-02-08T06:47:10.621Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826679074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195322024-02-08T06:47:10.621ZWait for a query_work_queue command to finish before sending IO
195332024-02-08T06:47:10.621Z{"msg":"[1]R flush_numbers[0..12]: [300, 300, 304, 304, 301, 300, 301, 298, 277, 304, 297, 301]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826720431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195342024-02-08T06:47:10.625Z{"msg":"[1]R generation[0..12]: [177, 177, 179, 179, 178, 177, 178, 176, 162, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826759718Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195352024-02-08T06:47:10.625Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826788716Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195362024-02-08T06:47:10.625Z{"msg":"[2]R flush_numbers[0..12]: [300, 300, 304, 304, 301, 300, 301, 298, 277, 304, 297, 301]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826816054Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195372024-02-08T06:47:10.625Z{"msg":"[2]R generation[0..12]: [177, 177, 179, 179, 178, 177, 178, 176, 162, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826855461Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195382024-02-08T06:47:10.625Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826910366Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195392024-02-08T06:47:10.625Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826950514Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195402024-02-08T06:47:10.625Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.826978432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195412024-02-08T06:47:10.625Z{"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.827031108Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195422024-02-08T06:47:10.625Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.827069155Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195432024-02-08T06:47:10.625Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.827103242Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195442024-02-08T06:47:10.625Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.827119361Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195452024-02-08T06:47:10.625Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.827140459Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195462024-02-08T06:47:10.626Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.827154308Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195472024-02-08T06:47:10.626Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
195482024-02-08T06:47:10.626Z"time":"2024-02-08T06:47:09.827174667Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195492024-02-08T06:47:10.626Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.827191816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195502024-02-08T06:47:10.626Z{"msg":"64c5ac96-05bd-48a6-97c0-35b78366bf40 is now active with session: 88ac6fa7-60bf-41f6-b72b-3069329efe3b","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.827211664Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195512024-02-08T06:47:10.626Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.827224373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858,"session_id":"88ac6fa7-60bf-41f6-b72b-3069329efe3b"}
195522024-02-08T06:47:10.626Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:09.827243741Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858}
195532024-02-08T06:47:10.626ZLoading write count information from file "/tmp/test_repair_verify.data"
195542024-02-08T06:47:10.626ZRun Repair workload
195552024-02-08T06:47:10.626Z01/30 Write block 83 len 40960 data: 16 16 16 15 14 13 12 13 16 19
195562024-02-08T06:47:10.626Z02/30 Read block 490 len 28672
195572024-02-08T06:47:10.626Z03/30 Write block 117 len 40960 data: 13 12 12 11 10 9 10 9 12 9
195582024-02-08T06:47:10.626Z04/30 Flush
195592024-02-08T06:47:10.626Z05/30 Write block 532 len 32768 data: 17 15 15 16 16 15 15 15
195602024-02-08T06:47:10.633Z06/30 Read block 214 len 8192
195612024-02-08T06:47:10.633Z07/30 Read block 495 len 12288
195622024-02-08T06:47:10.633Z08/30 Read block 265 len 16384
195632024-02-08T06:47:10.633Z09/30 Write block 269 len 32768 data: 17 11 9 11 13 12 15 17
195642024-02-08T06:47:10.633Z10/30 Read block 345 len 24576
195652024-02-08T06:47:10.633Z11/30 Write block 465 len 20480 data: 11 10 12 13 13
195662024-02-08T06:47:10.639Z12/30 Write block 427 len 28672 data: 8 8 8 12 11 12 9
195672024-02-08T06:47:10.639Z13/30 Flush
195682024-02-08T06:47:10.639Z14/30 Write block 506 len 40960 data: 7 8 8 11 14 11 10 12 10 9
195692024-02-08T06:47:10.639Z15/30 Write block 262 len 12288 data: 21 17 16
195702024-02-08T06:47:10.639Z16/30 Write block 237 len 16384 data: 13 18 19 16
195712024-02-08T06:47:10.640Z17/30 Read block 390 len 24576
195722024-02-08T06:47:10.640Z18/30 Write block 451 len 8192 data: 12 13
195732024-02-08T06:47:10.640Z19/30 Read block 35 len 24576
195742024-02-08T06:47:10.643Z20/30 Read block 243 len 28672
195752024-02-08T06:47:10.643Z21/30 Read block 427 len 24576
195762024-02-08T06:47:10.643Z22/30 Flush
195772024-02-08T06:47:10.643Z23/30 Write block 458 len 24576 data: 17 16 14 14 17 16
195782024-02-08T06:47:10.643Z24/30 Read block 21 len 4096
195792024-02-08T06:47:10.643Z25/30 Read block 265 len 36864
195802024-02-08T06:47:10.646Z26/30 Read block 121 len 40960
195812024-02-08T06:47:10.647Z27/30 Write block 430 len 20480 data: 13 12 13 10 12
195822024-02-08T06:47:10.647Z28/30 Read block 32 len 28672
195832024-02-08T06:47:10.650Z29/30 Write block 20 len 12288 data: 16 13 13
195842024-02-08T06:47:10.650Z30/30 Write block 566 len 40960 data: 13 14 15 12 14 15 14 14 11 11
195852024-02-08T06:47:10.650Z----------------------------------------------------------------
195862024-02-08T06:47:10.650Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
195872024-02-08T06:47:10.650ZStates: Active Active Active
195882024-02-08T06:47:10.650ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
195892024-02-08T06:47:10.650Z 1 Acked 1000 Write 10 Done Done Done false
195902024-02-08T06:47:10.650Z 2 Acked 1001 Read 7 Done Done Done false
195912024-02-08T06:47:10.650Z 3 Acked 1002 Write 10 Done Done Sent false
195922024-02-08T06:47:10.650Z 4 Acked 1003 Flush 0 Done Done Sent false
195932024-02-08T06:47:10.651Z 5 Acked 1004 Write 8 Done Done Sent false
195942024-02-08T06:47:10.651Z 6 Acked 1005 Read 2 Done Done Sent false
195952024-02-08T06:47:10.651Z 7 Acked 1006 Read 3 Done Done Sent false
195962024-02-08T06:47:10.651Z 8 Acked 1007 Read 4 Done Done Sent false
195972024-02-08T06:47:10.651Z 9 Acked 1008 Write 8 Done Done Sent false
195982024-02-08T06:47:10.651Z 10 Acked 1009 Read 6 Done Done Sent false
195992024-02-08T06:47:10.651Z 11 Acked 1010 Write 5 Done Done Sent false
196002024-02-08T06:47:10.651Z 12 Acked 1011 Write 7 Done Done Sent false
196012024-02-08T06:47:10.651Z 13 Acked 1012 Flush 0 Done Done Sent false
196022024-02-08T06:47:10.651Z 14 Acked 1013 Write 10 Done Done Sent false
196032024-02-08T06:47:10.651Z 15 Acked 1014 Write 3 Done Done Sent false
196042024-02-08T06:47:10.651Z 16 Acked 1015 Write 4 Done Done Sent false
196052024-02-08T06:47:10.651Z 17 Acked 1016 Read 6 Done Done Sent false
196062024-02-08T06:47:10.651Z 18 Acked 1017 Write 2 Done Done Sent false
196072024-02-08T06:47:10.651Z 19 Acked 1018 Read 6 Done Done Sent false
196082024-02-08T06:47:10.651Z 20 Acked 1019 Read 7 Done Done Sent false
196092024-02-08T06:47:10.652Z 21 Acked 1020 Read 6 Done Done Sent false
196102024-02-08T06:47:10.652Z 22 Acked 1021 Flush 0 Done Done Sent false
196112024-02-08T06:47:10.652Z 23 Acked 1022 Write 6 Done Done Sent false
196122024-02-08T06:47:10.652Z 24 Acked 1023 Read 1 Done Done Sent false
196132024-02-08T06:47:10.652Z 25 Acked 1024 Read 9 Done Done Sent false
196142024-02-08T06:47:10.652Z 26 Acked 1025 Read 10 Done Done Sent false
196152024-02-08T06:47:10.652Z 27 Acked 1026 Write 5 Done Done Sent false
196162024-02-08T06:47:10.652Z 28 Acked 1027 Read 7 Done Done Sent false
196172024-02-08T06:47:10.652Z 29 Acked 1028 Write 3 Done Done Sent false
196182024-02-08T06:47:10.652Z 30 Acked 1029 Write 10 Sent Sent Sent false
196192024-02-08T06:47:10.652Z STATES DS:0 DS:1 DS:2 TOTAL
196202024-02-08T06:47:10.652Z New 0 0 0 0
196212024-02-08T06:47:10.652Z Sent 1 1 28 30
196222024-02-08T06:47:10.652Z Done 29 29 2 60
196232024-02-08T06:47:10.652Z Skipped 0 0 0 0
196242024-02-08T06:47:10.653Z Error 0 0 0 0
196252024-02-08T06:47:10.653ZLast Flush: 1021 1021 0
196262024-02-08T06:47:10.653ZDownstairs last five completed:
196272024-02-08T06:47:10.653ZUpstairs last five completed: 30 29 28 27 26
196282024-02-08T06:47:10.653Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:09.859529997Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1858}
196292024-02-08T06:47:10.653ZWrote out file "/tmp/test_repair_verify.data"
196302024-02-08T06:47:10.656Z
196312024-02-08T06:47:10.656ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196322024-02-08T06:47:10.673ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196332024-02-08T06:47:10.673Z 0 000-019 177 177 177 300 300 300 F F F
196342024-02-08T06:47:10.673Z 1 020-039 177 177 177 300 300 300 T T F <---
196352024-02-08T06:47:10.673Z 2 040-059 179 179 179 304 304 304 F F F
196362024-02-08T06:47:10.673Z 3 060-079 179 179 179 304 304 304 F F F
196372024-02-08T06:47:10.673Z 4 080-099 180 180 178 305 305 301 F F T <---
196382024-02-08T06:47:10.673Z 5 100-119 180 180 177 305 305 300 F F F <---
196392024-02-08T06:47:10.673Z 6 120-139 180 180 178 305 305 301 F F F <---
196402024-02-08T06:47:10.673Z 7 140-159 176 176 176 298 298 298 F F F
196412024-02-08T06:47:10.677Z 8 160-179 162 162 162 277 277 277 F F F
196422024-02-08T06:47:10.677Z 9 180-199 179 179 179 304 304 304 F F F
196432024-02-08T06:47:10.677Z 10 200-219 174 174 174 297 297 297 F F F
196442024-02-08T06:47:10.677Z 11 220-239 180 180 178 307 307 301 F F F <---
196452024-02-08T06:47:10.677Z 12 240-259 180 180 177 307 307 300 F F F <---
196462024-02-08T06:47:10.677Z 13 260-279 180 180 179 307 307 304 F F F <---
196472024-02-08T06:47:10.677Z 14 280-299 179 179 179 304 304 304 F F F
196482024-02-08T06:47:10.677Z 15 300-319 178 178 178 301 301 301 F F F
196492024-02-08T06:47:10.677Z 16 320-339 179 179 179 304 304 304 F F F
196502024-02-08T06:47:10.677Z 17 340-359 178 178 178 303 303 303 F F F
196512024-02-08T06:47:10.677Z 18 360-379 178 178 178 303 303 303 F F F
196522024-02-08T06:47:10.677Z 19 380-399 179 179 179 304 304 304 F F F
196532024-02-08T06:47:10.678Z 20 400-419 179 179 179 304 304 304 F F F
196542024-02-08T06:47:10.678Z 21 420-439 180 180 179 306 306 304 T T F <---
196552024-02-08T06:47:10.678Z 22 440-459 180 180 172 307 307 295 T T F <---
196562024-02-08T06:47:10.678Z 23 460-479 180 180 179 306 306 304 T T F <---
196572024-02-08T06:47:10.678Z 24 480-499 179 179 179 304 304 304 F F F
196582024-02-08T06:47:10.678Z 25 500-519 180 180 178 307 307 301 F F F <---
196592024-02-08T06:47:10.678Z 26 520-539 180 180 178 306 306 303 F F F <---
196602024-02-08T06:47:10.678Z 27 540-559 178 178 178 301 301 301 F F F
196612024-02-08T06:47:10.678Z 28 560-579 169 169 169 288 288 288 T T F <---
196622024-02-08T06:47:10.678Z 29 580-599 174 174 174 297 297 297 F F F
196632024-02-08T06:47:10.678ZMax gen: 180, Max flush: 307
196642024-02-08T06:47:10.678ZError: Difference in extent metadata found!
196652024-02-08T06:47:10.678ZA Difference in extent metadata is expected here
196662024-02-08T06:47:10.678ZOn loop 90
196672024-02-08T06:47:10.679Z
196682024-02-08T06:47:10.682ZVerifying data now
196692024-02-08T06:47:15.309ZNo change after verify
196702024-02-08T06:47:15.309ZLoop: 90 Downstairs dump after verify (and repair):
196712024-02-08T06:47:15.328ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196722024-02-08T06:47:15.328Z 0 000-019 177 177 177 300 300 300 F F F
196732024-02-08T06:47:15.770Z 1 020-039 181 181 181 308 308 308 F F F
196742024-02-08T06:47:15.771Z 2 040-059 179 179 179 304 304 304 F F F
196752024-02-08T06:47:15.771Z 3 060-079 179 179 179 304 304 304 F F F
196762024-02-08T06:47:15.771Z 4 080-099 180 180 180 305 305 305 F F F
196772024-02-08T06:47:15.771Z 5 100-119 180 180 180 305 305 305 F F F
196782024-02-08T06:47:15.771Z 6 120-139 180 180 180 305 305 305 F F F
196792024-02-08T06:47:15.771Z 7 140-159 176 176 176 298 298 298 F F F
196802024-02-08T06:47:15.771Z 8 160-179 162 162 162 277 277 277 F F F
196812024-02-08T06:47:15.771Z 9 180-199 179 179 179 304 304 304 F F F
196822024-02-08T06:47:15.771Z 10 200-219 174 174 174 297 297 297 F F F
196832024-02-08T06:47:15.771Z 11 220-239 180 180 180 307 307 307 F F F
196842024-02-08T06:47:15.771Z 12 240-259 180 180 180 307 307 307 F F F
196852024-02-08T06:47:15.771Z 13 260-279 180 180 180 307 307 307 F F F
196862024-02-08T06:47:15.771Z 14 280-299 179 179 179 304 304 304 F F F
196872024-02-08T06:47:15.772Z 15 300-319 178 178 178 301 301 301 F F F
196882024-02-08T06:47:15.772Z 16 320-339 179 179 179 304 304 304 F F F
196892024-02-08T06:47:15.772Z 17 340-359 178 178 178 303 303 303 F F F
196902024-02-08T06:47:15.772Z 18 360-379 178 178 178 303 303 303 F F F
196912024-02-08T06:47:15.772Z 19 380-399 179 179 179 304 304 304 F F F
196922024-02-08T06:47:15.772Z 20 400-419 179 179 179 304 304 304 F F F
196932024-02-08T06:47:15.772Z 21 420-439 181 181 181 308 308 308 F F F
196942024-02-08T06:47:15.772Z 22 440-459 181 181 181 308 308 308 F F F
196952024-02-08T06:47:15.772Z 23 460-479 181 181 181 308 308 308 F F F
196962024-02-08T06:47:15.772Z 24 480-499 179 179 179 304 304 304 F F F
196972024-02-08T06:47:15.772Z 25 500-519 180 180 180 307 307 307 F F F
196982024-02-08T06:47:15.772Z 26 520-539 180 180 180 306 306 306 F F F
196992024-02-08T06:47:15.772Z 27 540-559 178 178 178 301 301 301 F F F
197002024-02-08T06:47:15.772Z 28 560-579 181 181 181 308 308 308 F F F
197012024-02-08T06:47:15.772Z 29 580-599 174 174 174 297 297 297 F F F
197022024-02-08T06:47:15.773ZMax gen: 181, Max flush: 308
197032024-02-08T06:47:15.773Z
197042024-02-08T06:47:15.773ZBegin loop 91
197052024-02-08T06:47:15.773ZDownstairs to restart: 0
197062024-02-08T06:47:15.773Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.55607777Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866}
197072024-02-08T06:47:15.773Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556325482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866}
197082024-02-08T06:47:15.773Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.55634683Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866}
197092024-02-08T06:47:15.773Z{"msg":"Crucible 2d3f94f4-7db3-47cb-af64-243a9d426151 has session id: c5628caf-f5d2-4bf5-b554-1f11bb781217","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556379388Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197102024-02-08T06:47:15.773Z{"msg":"Upstairs opts: Upstairs UUID: 2d3f94f4-7db3-47cb-af64-243a9d426151, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556406046Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197112024-02-08T06:47:15.773Z{"msg":"Crucible stats registered with UUID: 2d3f94f4-7db3-47cb-af64-243a9d426151","v":0,"name":"crucible","level":30Crucible runtime is spawned
197122024-02-08T06:47:15.773Z,"time":"2024-02-08T06:47:14.556442523Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197132024-02-08T06:47:15.773Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556532226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866}
197142024-02-08T06:47:15.773Z{"msg":"2d3f94f4-7db3-47cb-af64-243a9d426151 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556600731Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197152024-02-08T06:47:15.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556638538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197162024-02-08T06:47:15.773Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556683415Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197172024-02-08T06:47:15.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556725542Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197182024-02-08T06:47:15.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.55675428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197192024-02-08T06:47:15.774Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556779538Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197202024-02-08T06:47:15.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556821545Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197212024-02-08T06:47:15.774Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556864911Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"io task","client":"0","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197222024-02-08T06:47:15.774Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556892789Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"io task","client":"1","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197232024-02-08T06:47:15.774Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556918957Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197242024-02-08T06:47:15.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556957544Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"io task","client":"0","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197252024-02-08T06:47:15.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.556983662Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"io task","client":"1","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197262024-02-08T06:47:15.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.5570188Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197272024-02-08T06:47:15.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.557628194Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197282024-02-08T06:47:15.774Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.557670811Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197292024-02-08T06:47:15.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.557695799Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197302024-02-08T06:47:15.774Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.557725397Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197312024-02-08T06:47:15.774Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.558228799Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197322024-02-08T06:47:15.774Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.558370708Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197332024-02-08T06:47:15.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.558758909Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197342024-02-08T06:47:15.775Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.558807176Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197352024-02-08T06:47:15.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.55888173Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197362024-02-08T06:47:15.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.558924937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197372024-02-08T06:47:15.775Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.558954525Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197382024-02-08T06:47:15.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.559026439Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197392024-02-08T06:47:15.775Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.559041488Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197402024-02-08T06:47:15.775Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.559063576Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197412024-02-08T06:47:15.775Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.559491464Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197422024-02-08T06:47:15.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.5599456Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197432024-02-08T06:47:15.775Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.559981697Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197442024-02-08T06:47:15.775Z{"msg":"[0]R flush_numbers[0..12]: [300, 308, 304, 304, 305, 305, 305, 298, 277, 304, 297, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560034133Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197452024-02-08T06:47:15.775Z{"msg":"[0]R generation[0..12]: [177, 181, 179, 179, 180, 180, 180, 176, 162, 179, 174, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560070031Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197462024-02-08T06:47:15.775Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560103748Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197472024-02-08T06:47:15.775ZWait for a query_work_queue command to finish before sending IO
197482024-02-08T06:47:15.776Z{"msg":"[1]R flush_numbers[0..12]: [300, 308, 304, 304, 305, 305, 305, 298, 277, 304, 297, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560138425Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197492024-02-08T06:47:15.776Z{"msg":"[1]R generation[0..12]: [177, 181, 179, 179, 180, 180, 180, 176, 162, 179, 174, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560171593Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197502024-02-08T06:47:15.776Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.56020534Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197512024-02-08T06:47:15.776Z{"msg":"[2]R flush_numbers[0..12]: [300, 308, 304, 304, 305, 305, 305, 298, 277, 304, 297, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560232858Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197522024-02-08T06:47:15.776Z{"msg":"[2]R generation[0..12]: [177, 181, 179, 179, 180, 180, 180, 176, 162, 179, 174, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560266376Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197532024-02-08T06:47:15.776Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560292774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197542024-02-08T06:47:15.776Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560321122Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197552024-02-08T06:47:15.776Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560336001Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197562024-02-08T06:47:15.776Z{"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560356229Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197572024-02-08T06:47:15.776Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560370068Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197582024-02-08T06:47:15.776Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560396176Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197592024-02-08T06:47:15.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560408685Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197602024-02-08T06:47:15.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560429574Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197612024-02-08T06:47:15.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560443333Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197622024-02-08T06:47:15.776Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560463321Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197632024-02-08T06:47:15.776Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.56047628Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197642024-02-08T06:47:15.776Z{"msg":"2d3f94f4-7db3-47cb-af64-243a9d426151 is now active with session: c5628caf-f5d2-4bf5-b554-1f11bb781217","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560489529Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197652024-02-08T06:47:15.776Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560507868Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866,"session_id":"c5628caf-f5d2-4bf5-b554-1f11bb781217"}
197662024-02-08T06:47:15.777Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:14.560520407Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197672024-02-08T06:47:15.777Zip-10-150-1-117.us-west-2.compute.internal","pid":1866}
197682024-02-08T06:47:15.777ZLoading write count information from file "/tmp/test_repair_verify.data"
197692024-02-08T06:47:15.777ZRun Repair workload
197702024-02-08T06:47:15.777Z01/30 Write block 258 len 16384 data: 12 15 15 21
197712024-02-08T06:47:15.777Z02/30 Read block 361 len 40960
197722024-02-08T06:47:15.777Z03/30 Write block 12 len 32768 data: 13 14 13 13 12 12 13 14
197732024-02-08T06:47:15.825Z04/30 Read block 293 len 20480
197742024-02-08T06:47:15.825Z05/30 Read block 531 len 36864
197752024-02-08T06:47:15.825Z06/30 Flush
197762024-02-08T06:47:15.825Z07/30 Flush
197772024-02-08T06:47:15.826Z08/30 Write block 499 len 32768 data: 12 13 10 8 6 9 7 8
197782024-02-08T06:47:15.826Z09/30 Write block 477 len 12288 data: 11 9 10
197792024-02-08T06:47:15.826Z10/30 Read block 137 len 16384
197802024-02-08T06:47:15.826Z11/30 Read block 563 len 8192
197812024-02-08T06:47:15.826Z12/30 Read block 245 len 36864
197822024-02-08T06:47:15.826Z13/30 Read block 10 len 40960
197832024-02-08T06:47:15.826Z14/30 Write block 417 len 36864 data: 14 17 17 16 16 14 14 13 11
197842024-02-08T06:47:15.826Z15/30 Write block 553 len 24576 data: 26 23 21 23 15 11
197852024-02-08T06:47:15.826Z16/30 Flush
197862024-02-08T06:47:15.826Z17/30 Write block 166 len 20480 data: 20 22 16 16 16
197872024-02-08T06:47:15.826Z18/30 Read block 567 len 20480
197882024-02-08T06:47:15.827Z19/30 Read block 439 len 8192
197892024-02-08T06:47:15.827Z20/30 Read block 591 len 16384
197902024-02-08T06:47:15.827Z21/30 Write block 214 len 4096 data: 20
197912024-02-08T06:47:15.827Z22/30 Write block 330 len 24576 data: 9 13 15 17 20 20
197922024-02-08T06:47:15.827Z23/30 Write block 27 len 16384 data: 18 17 15 17
197932024-02-08T06:47:15.827Z24/30 Write block 264 len 32768 data: 17 17 17 18 18 18 12 10
197942024-02-08T06:47:15.827Z25/30 Write block 522 len 36864 data: 10 11 13 13 13 13 14 14 13
197952024-02-08T06:47:15.827Z26/30 Read block 135 len 32768
197962024-02-08T06:47:15.827Z27/30 Read block 471 len 24576
197972024-02-08T06:47:15.827Z28/30 Write block 164 len 12288 data: 17 19 21
197982024-02-08T06:47:15.827Z29/30 Write block 318 len 12288 data: 11 13 18
197992024-02-08T06:47:15.828Z30/30 Write block 77 len 28672 data: 12 12 13 14 14 16 17
198002024-02-08T06:47:15.828Z----------------------------------------------------------------
198012024-02-08T06:47:15.828Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
198022024-02-08T06:47:15.828ZStates: Active Active Active
198032024-02-08T06:47:15.828ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198042024-02-08T06:47:15.828Z 1 Acked 1000 Write 4 Done Done Done false
198052024-02-08T06:47:15.828Z 2 Acked 1001 Read 10 Done Done Done false
198062024-02-08T06:47:15.828Z 3 Acked 1002 Write 8 Sent Done Done false
198072024-02-08T06:47:15.828Z 4 Acked 1003 Read 5 Sent Done Done false
198082024-02-08T06:47:15.828Z 5 Acked 1004 Read 9 Sent Done Done false
198092024-02-08T06:47:15.828Z 6 Acked 1005 Flush 0 Sent Done Done false
198102024-02-08T06:47:15.829Z 7 Acked 1006 Flush 0 Sent Done Done false
198112024-02-08T06:47:15.829Z 8 Acked 1007 Write 8 Sent Done Done false
198122024-02-08T06:47:15.829Z 9 Acked 1008 Write 3 Sent Done Done false
198132024-02-08T06:47:15.829Z 10 Acked 1009 Read 4 Sent Done Done false
198142024-02-08T06:47:15.829Z 11 Acked 1010 Read 2 Sent Done Done false
198152024-02-08T06:47:15.829Z 12 Acked 1011 Read 9 Sent Done Done false
198162024-02-08T06:47:15.829Z 13 Acked 1012 Read 10 Sent Done Done false
198172024-02-08T06:47:15.829Z 14 Acked 1013 Write 9 Sent Done Done false
198182024-02-08T06:47:15.829Z 15 Acked 1014 Write 6 Sent Done Done false
198192024-02-08T06:47:15.829Z 16 Acked 1015 Flush 0 Sent Done Done false
198202024-02-08T06:47:15.829Z 17 Acked 1016 Write 5 Sent Done Done false
198212024-02-08T06:47:15.830Z 18 Acked 1017 Read 5 Sent Done Done false
198222024-02-08T06:47:15.830Z 19 Acked 1018 Read 2 Sent Done Done false
198232024-02-08T06:47:15.830Z 20 Acked 1019 Read 4 Sent Done Done false
198242024-02-08T06:47:15.830Z 21 Acked 1020 Write 1 Sent Done Done false
198252024-02-08T06:47:15.830Z 22 Acked 1021 Write 6 Sent Done Done false
198262024-02-08T06:47:15.830Z 23 Acked 1022 Write 4 Sent Done Done false
198272024-02-08T06:47:15.830Z 24 Acked 1023 Write 8 Sent Done Done false
198282024-02-08T06:47:15.830Z 25 Acked 1024 Write 9 Sent Done Done false
198292024-02-08T06:47:15.830Z 26 Acked 1025 Read 8 Sent Done Done false
198302024-02-08T06:47:15.830Z 27 Acked 1026 Read 6 Sent Done Done false
198312024-02-08T06:47:15.830Z 28 Acked 1027 Write 3 Sent Done Done false
198322024-02-08T06:47:15.830Z 29 Acked 1028 Write 3 Sent Done Done false
198332024-02-08T06:47:15.831Z 30 Acked 1029 Write 7 Sent Sent Sent false
198342024-02-08T06:47:15.831Z STATES DS:0 DS:1 DS:2 TOTAL
198352024-02-08T06:47:15.831Z New 0 0 0 0
198362024-02-08T06:47:15.831Z Sent 28 1 1 30
198372024-02-08T06:47:15.831Z Done 2 29 29 60
198382024-02-08T06:47:15.831Z Skipped 0 0 0 0
198392024-02-08T06:47:15.831Z Error 0 0 0 0
198402024-02-08T06:47:15.831ZLast Flush: 0 1015 1015
198412024-02-08T06:47:15.831ZDownstairs last five completed:
198422024-02-08T06:47:15.831ZUpstairs last five completed: 30 29 28 27 26
198432024-02-08T06:47:15.831Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:14.589714795Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1866}
198442024-02-08T06:47:15.832ZWrote out file "/tmp/test_repair_verify.data"
198452024-02-08T06:47:15.832Z
198462024-02-08T06:47:15.832ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198472024-02-08T06:47:15.832ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198482024-02-08T06:47:15.832Z 0 000-019 177 182 182 300 309 309 F F F <---
198492024-02-08T06:47:15.832Z 1 020-039 181 181 181 308 308 308 F T T <---
198502024-02-08T06:47:15.832Z 2 040-059 179 179 179 304 304 304 F F F
198512024-02-08T06:47:15.832Z 3 060-079 179 179 179 304 304 304 F T T <---
198522024-02-08T06:47:15.832Z 4 080-099 180 180 180 305 305 305 F T T <---
198532024-02-08T06:47:15.832Z 5 100-119 180 180 180 305 305 305 F F F
198542024-02-08T06:47:15.832Z 6 120-139 180 180 180 305 305 305 F F F
198552024-02-08T06:47:15.833Z 7 140-159 176 176 176 298 298 298 F F F
198562024-02-08T06:47:15.833Z 8 160-179 162 162 162 277 277 277 F T T <---
198572024-02-08T06:47:15.833Z 9 180-199 179 179 179 304 304 304 F F F
198582024-02-08T06:47:15.833Z 10 200-219 174 174 174 297 297 297 F T T <---
198592024-02-08T06:47:15.833Z 11 220-239 180 180 180 307 307 307 F F F
198602024-02-08T06:47:15.833Z 12 240-259 180 182 182 307 309 309 T F F <---
198612024-02-08T06:47:15.833Z 13 260-279 180 182 182 307 309 309 T T T <---
198622024-02-08T06:47:15.833Z 14 280-299 179 179 179 304 304 304 F F F
198632024-02-08T06:47:15.833Z 15 300-319 178 178 178 301 301 301 F T T <---
198642024-02-08T06:47:15.834Z 16 320-339 179 179 179 304 304 304 F T T <---
198652024-02-08T06:47:15.834Z 17 340-359 178 178 178 303 303 303 F F F
198662024-02-08T06:47:15.834Z 18 360-379 178 178 178 303 303 303 F F F
198672024-02-08T06:47:15.834Z 19 380-399 179 179 179 304 304 304 F F F
198682024-02-08T06:47:15.834Z 20 400-419 179 182 182 304 311 311 F F F <---
198692024-02-08T06:47:15.834Z 21 420-439 181 182 182 308 311 311 F F F <---
198702024-02-08T06:47:15.835Z 22 440-459 181 181 181 308 308 308 F F F
198712024-02-08T06:47:15.835Z 23 460-479 181 182 182 308 311 311 F F F <---
198722024-02-08T06:47:15.835Z 24 480-499 179 182 182 304 311 311 F F F <---
198732024-02-08T06:47:15.848Z 25 500-519 180 182 182 307 311 311 F F F <---
198742024-02-08T06:47:15.848Z 26 520-539 180 180 180 306 306 306 F T T <---
198752024-02-08T06:47:15.849Z 27 540-559 178 182 182 301 311 311 F F F <---
198762024-02-08T06:47:15.849Z 28 560-579 181 181 181 308 308 308 F F F
198772024-02-08T06:47:15.849Z 29 580-599 174 174 174 297 297 297 F F F
198782024-02-08T06:47:15.849ZMax gen: 182, Max flush: 311
198792024-02-08T06:47:15.850ZError: Difference in extent metadata found!
198802024-02-08T06:47:15.850ZA Difference in extent metadata is expected here
198812024-02-08T06:47:15.850ZOn loop 91
198822024-02-08T06:47:15.850Z
198832024-02-08T06:47:15.851ZVerifying data now
198842024-02-08T06:47:20.220ZNo change after verify
198852024-02-08T06:47:20.220ZLoop: 91 Downstairs dump after verify (and repair):
198862024-02-08T06:47:20.248ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198872024-02-08T06:47:20.248Z 0 000-019 182 182 182 309 309 309 F F F
198882024-02-08T06:47:20.671Z 1 020-039 183 183 183 312 312 312 F F F
198892024-02-08T06:47:20.671Z 2 040-059 179 179 179 304 304 304 F F F
198902024-02-08T06:47:20.671Z 3 060-079 183 183 183 312 312 312 F F F
198912024-02-08T06:47:20.671Z 4 080-099 183 183 183 312 312 312 F F F
198922024-02-08T06:47:20.671Z 5 100-119 180 180 180 305 305 305 F F F
198932024-02-08T06:47:20.671Z 6 120-139 180 180 180 305 305 305 F F F
198942024-02-08T06:47:20.671Z 7 140-159 176 176 176 298 298 298 F F F
198952024-02-08T06:47:20.671Z 8 160-179 183 183 183 312 312 312 F F F
198962024-02-08T06:47:20.672Z 9 180-199 179 179 179 304 304 304 F F F
198972024-02-08T06:47:20.672Z 10 200-219 183 183 183 312 312 312 F F F
198982024-02-08T06:47:20.672Z 11 220-239 180 180 180 307 307 307 F F F
198992024-02-08T06:47:20.672Z 12 240-259 182 182 182 309 309 309 F F F
199002024-02-08T06:47:20.672Z 13 260-279 183 183 183 312 312 312 F F F
199012024-02-08T06:47:20.672Z 14 280-299 179 179 179 304 304 304 F F F
199022024-02-08T06:47:20.672Z 15 300-319 183 183 183 312 312 312 F F F
199032024-02-08T06:47:20.672Z 16 320-339 183 183 183 312 312 312 F F F
199042024-02-08T06:47:20.672Z 17 340-359 178 178 178 303 303 303 F F F
199052024-02-08T06:47:20.672Z 18 360-379 178 178 178 303 303 303 F F F
199062024-02-08T06:47:20.673Z 19 380-399 179 179 179 304 304 304 F F F
199072024-02-08T06:47:20.673Z 20 400-419 182 182 182 311 311 311 F F F
199082024-02-08T06:47:20.673Z 21 420-439 182 182 182 311 311 311 F F F
199092024-02-08T06:47:20.673Z 22 440-459 181 181 181 308 308 308 F F F
199102024-02-08T06:47:20.673Z 23 460-479 182 182 182 311 311 311 F F F
199112024-02-08T06:47:20.673Z 24 480-499 182 182 182 311 311 311 F F F
199122024-02-08T06:47:20.673Z 25 500-519 182 182 182 311 311 311 F F F
199132024-02-08T06:47:20.673Z 26 520-539 183 183 183 312 312 312 F F F
199142024-02-08T06:47:20.673Z 27 540-559 182 182 182 311 311 311 F F F
199152024-02-08T06:47:20.673Z 28 560-579 181 181 181 308 308 308 F F F
199162024-02-08T06:47:20.673Z 29 580-599 174 174 174 297 297 297 F F F
199172024-02-08T06:47:20.673ZMax gen: 183, Max flush: 312
199182024-02-08T06:47:20.673Z
199192024-02-08T06:47:20.674ZBegin loop 92
199202024-02-08T06:47:20.674ZDownstairs to restart: 2
199212024-02-08T06:47:20.674Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.461186926Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874}
199222024-02-08T06:47:20.674Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.461436287Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874}
199232024-02-08T06:47:20.674Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.461453306Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874}
199242024-02-08T06:47:20.674Z{"msg":"Crucible 83042dcd-5702-4284-ae07-e0fac661f13d has session id: 642be245-e4a0-4ae5-8d6e-06d701a366ff","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.461480754Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199252024-02-08T06:47:20.674Z{"msg":"Upstairs opts: Upstairs UUID: 83042dcd-5702-4284-ae07-e0fac661f13d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.461505592Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199262024-02-08T06:47:20.674Z{"msg":"Crucible stats registered with UUID: 83042dcd-5702-4284-ae07-e0fac661f13d","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.46153828Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199272024-02-08T06:47:20.674ZCrucible runtime is spawned
199282024-02-08T06:47:20.674Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.461692778Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874}
199292024-02-08T06:47:20.675Z{"msg":"83042dcd-5702-4284-ae07-e0fac661f13d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.461817879Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199302024-02-08T06:47:20.675Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.461860956Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199312024-02-08T06:47:20.675Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.461913472Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199322024-02-08T06:47:20.675Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.461983277Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199332024-02-08T06:47:20.675Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462019824Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199342024-02-08T06:47:20.675Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462053091Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199352024-02-08T06:47:20.675Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462116327Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199362024-02-08T06:47:20.676Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462166373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199372024-02-08T06:47:20.676Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.46220016Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199382024-02-08T06:47:20.676Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462246637Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199392024-02-08T06:47:20.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462276714Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199402024-02-08T06:47:20.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462308672Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199412024-02-08T06:47:20.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462347189Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199422024-02-08T06:47:20.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462894468Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199432024-02-08T06:47:20.677Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462930785Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199442024-02-08T06:47:20.677Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462948694Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199452024-02-08T06:47:20.677Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.462982571Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199462024-02-08T06:47:20.677Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.463476074Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199472024-02-08T06:47:20.677Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.463568487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199482024-02-08T06:47:20.677Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.463692128Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199492024-02-08T06:47:20.677Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.463739765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199502024-02-08T06:47:20.677Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.463954418Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199512024-02-08T06:47:20.678Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.464005645Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199522024-02-08T06:47:20.712Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.464038692Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199532024-02-08T06:47:20.713Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.464212369Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199542024-02-08T06:47:20.713Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.464253656Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199552024-02-08T06:47:20.713Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.464271015Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199562024-02-08T06:47:20.713Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.464494518Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199572024-02-08T06:47:20.713Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.464946954Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199582024-02-08T06:47:20.713Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.464989261Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199592024-02-08T06:47:20.713Z{"msg":"[0]R flush_numbers[0..12]: [309, 312, 304, 312, 312, 305, 305, 298, 312, 304, 312, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.46500663Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199602024-02-08T06:47:20.713Z{"msg":"[0]R generation[0..12]: [182, 183, 179, 183, 183, 180, 180, 176, 183, 179, 183, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465022568Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199612024-02-08T06:47:20.713Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465037747Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199622024-02-08T06:47:20.713Z{"msg":"[1]R flush_numbers[0..12]: [309, 312, 304, 312, 312, 305, 305, 298, 312, 304, 312, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465052326Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199632024-02-08T06:47:20.713Z{"msg":"[1]R generation[0..12]: [182, 183, 179, 183, 183, 180, 180, 176, 183, 179, 183, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465074594Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199642024-02-08T06:47:20.713Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465089443Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199652024-02-08T06:47:20.714Z{"msg":"[2]R flush_numbers[0..12]: [309, 312, 304, 312, 312, 305, 305, 298, 312, 304, 312, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465104382Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199662024-02-08T06:47:20.714Z{"msg":"[2]R generation[0..12]: [182, 183, 179, 183, 183, 180, 180, 176, 183, 179, 183, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465118331Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199672024-02-08T06:47:20.714Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.46514026Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199682024-02-08T06:47:20.715Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465154358Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199692024-02-08T06:47:20.715Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465175417Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199702024-02-08T06:47:20.715Z{"msg":"Wait for a query_work_queue command to finish before sending IO
199712024-02-08T06:47:20.715ZNext flush: 313","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465191136Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199722024-02-08T06:47:20.715Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465204345Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199732024-02-08T06:47:20.715Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465224903Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199742024-02-08T06:47:20.715Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465237582Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199752024-02-08T06:47:20.715Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465251801Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199762024-02-08T06:47:20.715Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465271559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199772024-02-08T06:47:20.715Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465286429Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199782024-02-08T06:47:20.715Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465299068Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199792024-02-08T06:47:20.715Z{"msg":"83042dcd-5702-4284-ae07-e0fac661f13d is now active with session: 642be245-e4a0-4ae5-8d6e-06d701a366ff","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465312287Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199802024-02-08T06:47:20.716Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465330995Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874,"session_id":"642be245-e4a0-4ae5-8d6e-06d701a366ff"}
199812024-02-08T06:47:20.716Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:19.465343704Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874}
199822024-02-08T06:47:20.716ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
199832024-02-08T06:47:20.716ZLoading write count information from file "/tmp/test_repair_verify.data"
199842024-02-08T06:47:20.716ZRun Repair workload
199852024-02-08T06:47:20.716Z01/30 Write block 380 len 12288 data: 12 10 13
199862024-02-08T06:47:20.716Z02/30 Read block 103 len 24576
199872024-02-08T06:47:20.716Z03/30 Write block 37 len 24576 data: 12 14 11 10 10 10
199882024-02-08T06:47:20.716Z04/30 Write block 213 len 36864 data: 21 21 18 19 19 17 18 17 17
199892024-02-08T06:47:20.716Z05/30 Read block 45 len 20480
199902024-02-08T06:47:20.716Z06/30 Flush
199912024-02-08T06:47:20.716Z07/30 Flush
199922024-02-08T06:47:20.716Z08/30 Write block 577 len 24576 data: 11 10 14 12 12 12
199932024-02-08T06:47:20.716Z09/30 Read block 450 len 20480
199942024-02-08T06:47:20.716Z10/30 Write block 364 len 24576 data: 14 15 14 17 14 15
199952024-02-08T06:47:20.717Z11/30 Read block 60 len 24576
199962024-02-08T06:47:20.717Z12/30 Read block 154 len 24576
199972024-02-08T06:47:20.717Z13/30 Write block 530 len 28672 data: 14 14 18 16 16 17 17
199982024-02-08T06:47:20.717Z14/30 Read block 468 len 24576
199992024-02-08T06:47:20.717Z15/30 Read block 464 len 24576
200002024-02-08T06:47:20.717Z16/30 Flush
200012024-02-08T06:47:20.717Z17/30 Write block 207 len 36864 data: 18 15 15 13 22 23 22 22 19
200022024-02-08T06:47:20.717Z18/30 Read block 72 len 40960
200032024-02-08T06:47:20.717Z19/30 Read block 353 len 28672
200042024-02-08T06:47:20.717Z20/30 Read block 494 len 16384
200052024-02-08T06:47:20.717Z21/30 Flush
200062024-02-08T06:47:20.717Z22/30 Write block 437 len 20480 data: 18 17 14 15 15
200072024-02-08T06:47:20.717Z23/30 Write block 80 len 20480 data: 15 15 17 18 17
200082024-02-08T06:47:20.717Z24/30 Read block 584 len 4096
200092024-02-08T06:47:20.717Z25/30 Read block 465 len 24576
200102024-02-08T06:47:20.717Z26/30 Read block 175 len 16384
200112024-02-08T06:47:20.718Z27/30 Read block 569 len 24576
200122024-02-08T06:47:20.718Z28/30 Read block 262 len 24576
200132024-02-08T06:47:20.718Z29/30 Write block 376 len 20480 data: 15 13 13 12 13
200142024-02-08T06:47:20.718Z30/30 Write block 426 len 8192 data: 11 9
200152024-02-08T06:47:20.718Z----------------------------------------------------------------
200162024-02-08T06:47:20.718Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
200172024-02-08T06:47:20.718ZStates: Active Active Active
200182024-02-08T06:47:20.718ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200192024-02-08T06:47:20.718Z 1 Acked 1000 Write 3 Done Done Sent false
200202024-02-08T06:47:20.718Z 2 Acked 1001 Read 6 Done Done Sent false
200212024-02-08T06:47:20.718Z 3 Acked 1002 Write 6 Done Done Sent false
200222024-02-08T06:47:20.718Z 4 Acked 1003 Write 9 Done Done Sent false
200232024-02-08T06:47:20.718Z 5 Acked 1004 Read 5 Done Done Sent false
200242024-02-08T06:47:20.718Z 6 Acked 1005 Flush 0 Done Done Sent false
200252024-02-08T06:47:20.718Z 7 Acked 1006 Flush 0 Done Done Sent false
200262024-02-08T06:47:20.718Z 8 Acked 1007 Write 6 Done Done Sent false
200272024-02-08T06:47:20.718Z 9 Acked 1008 Read 5 Done Done Sent false
200282024-02-08T06:47:20.719Z 10 Acked 1009 Write 6 Done Done Sent false
200292024-02-08T06:47:20.719Z 11 Acked 1010 Read 6 Done Done Sent false
200302024-02-08T06:47:20.719Z 12 Acked 1011 Read 6 Done Done Sent false
200312024-02-08T06:47:20.719Z 13 Acked 1012 Write 7 Done Done Sent false
200322024-02-08T06:47:20.719Z 14 Acked 1013 Read 6 Done Done Sent false
200332024-02-08T06:47:20.719Z 15 Acked 1014 Read 6 Done Done Sent false
200342024-02-08T06:47:20.719Z 16 Acked 1015 Flush 0 Done Done Sent false
200352024-02-08T06:47:20.719Z 17 Acked 1016 Write 9 Done Done Sent false
200362024-02-08T06:47:20.719Z 18 Acked 1017 Read 10 Done Done Sent false
200372024-02-08T06:47:20.719Z 19 Acked 1018 Read 7 Done Done Sent false
200382024-02-08T06:47:20.719Z 20 Acked 1019 Read 4 Done Done Sent false
200392024-02-08T06:47:20.719Z 21 Acked 1020 Flush 0 Done Done Sent false
200402024-02-08T06:47:20.719Z 22 Acked 1021 Write 5 Done Done Sent false
200412024-02-08T06:47:20.719Z 23 Acked 1022 Write 5 Done Done Sent false
200422024-02-08T06:47:20.719Z 24 Acked 1023 Read 1 Done Done Sent false
200432024-02-08T06:47:20.719Z 25 Acked 1024 Read 6 Done Done Sent false
200442024-02-08T06:47:20.719Z 26 Acked 1025 Read 4 Done Done Sent false
200452024-02-08T06:47:20.719Z 27 Acked 1026 Read 6 Done Done Sent false
200462024-02-08T06:47:20.719Z 28 Acked 1027 Read 6 Done Done Sent false
200472024-02-08T06:47:20.720Z 29 Acked 1028 Write 5 Sent Sent Sent false
200482024-02-08T06:47:20.720Z 30 Acked 1029 Write 2 Sent Sent Sent false
200492024-02-08T06:47:20.720Z STATES DS:0 DS:1 DS:2 TOTAL
200502024-02-08T06:47:20.720Z New 0 0 0 0
200512024-02-08T06:47:20.720Z Sent 2 2 30 34
200522024-02-08T06:47:20.724Z Done 28 28 0 56
200532024-02-08T06:47:20.724Z Skipped 0 0 0 0
200542024-02-08T06:47:20.724Z Error 0 0 0 0
200552024-02-08T06:47:20.724ZLast Flush: 1020 1020 0
200562024-02-08T06:47:20.724ZDownstairs last five completed:
200572024-02-08T06:47:20.724ZUpstairs last five completed: 30 29 28 27 26
200582024-02-08T06:47:20.725Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:19.496974079Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1874}
200592024-02-08T06:47:20.725ZWrote out file "/tmp/test_repair_verify.data"
200602024-02-08T06:47:20.725Z
200612024-02-08T06:47:20.725ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
200622024-02-08T06:47:20.725ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200632024-02-08T06:47:20.725Z 0 000-019 182 182 182 309 309 309 F F F
200642024-02-08T06:47:20.725Z 1 020-039 184 184 183 313 313 312 F F F <---
200652024-02-08T06:47:20.725Z 2 040-059 184 184 179 313 313 304 F F F <---
200662024-02-08T06:47:20.725Z 3 060-079 183 183 183 312 312 312 F F F
200672024-02-08T06:47:20.725Z 4 080-099 183 183 183 312 312 312 T T F <---
200682024-02-08T06:47:20.725Z 5 100-119 180 180 180 305 305 305 F F F
200692024-02-08T06:47:20.725Z 6 120-139 180 180 180 305 305 305 F F F
200702024-02-08T06:47:20.725Z 7 140-159 176 176 176 298 298 298 F F F
200712024-02-08T06:47:20.725Z 8 160-179 183 183 183 312 312 312 F F F
200722024-02-08T06:47:20.725Z 9 180-199 179 179 179 304 304 304 F F F
200732024-02-08T06:47:20.726Z 10 200-219 184 184 183 316 316 312 F F F <---
200742024-02-08T06:47:20.726Z 11 220-239 184 184 180 313 313 307 F F F <---
200752024-02-08T06:47:20.726Z 12 240-259 182 182 182 309 309 309 F F F
200762024-02-08T06:47:20.726Z 13 260-279 183 183 183 312 312 312 F F F
200772024-02-08T06:47:20.726Z 14 280-299 179 179 179 304 304 304 F F F
200782024-02-08T06:47:20.726Z 15 300-319 183 183 183 312 312 312 F F F
200792024-02-08T06:47:20.726Z 16 320-339 183 183 183 312 312 312 F F F
200802024-02-08T06:47:20.726Z 17 340-359 178 178 178 303 303 303 F F F
200812024-02-08T06:47:20.726Z 18 360-379 184 184 178 315 315 303 T T F <---
200822024-02-08T06:47:20.726Z 19 380-399 184 184 179 313 313 304 T T F <---
200832024-02-08T06:47:20.726Z 20 400-419 182 182 182 311 311 311 F F F
200842024-02-08T06:47:20.726Z 21 420-439 182 182 182 311 311 311 T T F <---
200852024-02-08T06:47:20.726Z 22 440-459 181 181 181 308 308 308 T T F <---
200862024-02-08T06:47:20.726Z 23 460-479 182 182 182 311 311 311 F F F
200872024-02-08T06:47:20.726Z 24 480-499 182 182 182 311 311 311 F F F
200882024-02-08T06:47:20.726Z 25 500-519 182 182 182 311 311 311 F F F
200892024-02-08T06:47:20.727Z 26 520-539 184 184 183 315 315 312 F F F <---
200902024-02-08T06:47:20.727Z 27 540-559 182 182 182 311 311 311 F F F
200912024-02-08T06:47:20.727Z 28 560-579 184 184 181 315 315 308 F F F <---
200922024-02-08T06:47:20.727Z 29 580-599 184 184 174 315 315 297 F F F <---
200932024-02-08T06:47:20.727ZMax gen: 184, Max flush: 316
200942024-02-08T06:47:20.727ZError: Difference in extent metadata found!
200952024-02-08T06:47:20.727ZA Difference in extent metadata is expected here
200962024-02-08T06:47:20.727ZOn loop 92
200972024-02-08T06:47:20.727Z
200982024-02-08T06:47:20.727ZVerifying data now
200992024-02-08T06:47:24.905ZNo change after verify
201002024-02-08T06:47:24.905ZLoop: 92 Downstairs dump after verify (and repair):
201012024-02-08T06:47:24.920ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201022024-02-08T06:47:24.920Z 0 000-019 182 182 182 309 309 309 F F F
201032024-02-08T06:47:24.920Z 1 020-039 184 184 184 313 313 313 F F F
201042024-02-08T06:47:24.920Z 2 040-059 184 184 184 313 313 313 F F F
201052024-02-08T06:47:24.920Z 3 060-079 183 183 183 312 312 312 F F F
201062024-02-08T06:47:24.920Z 4 080-099 185 185 185 317 317 317 F F F
201072024-02-08T06:47:24.920Z 5 100-119 180 180 180 305 305 305 F F F
201082024-02-08T06:47:24.923Z 6 120-139 180 180 180 305 305 305 F F F
201092024-02-08T06:47:24.923Z 7 140-159 176 176 176 298 298 298 F F F
201102024-02-08T06:47:24.923Z 8 160-179 183 183 183 312 312 312 F F F
201112024-02-08T06:47:24.924Z 9 180-199 179 179 179 304 304 304 F F F
201122024-02-08T06:47:24.924Z 10 200-219 184 184 184 316 316 316 F F F
201132024-02-08T06:47:24.924Z 11 220-239 184 184 184 313 313 313 F F F
201142024-02-08T06:47:24.924Z 12 240-259 182 182 182 309 309 309 F F F
201152024-02-08T06:47:24.924Z 13 260-279 183 183 183 312 312 312 F F F
201162024-02-08T06:47:24.924Z 14 280-299 179 179 179 304 304 304 F F F
201172024-02-08T06:47:24.924Z 15 300-319 183 183 183 312 312 312 F F F
201182024-02-08T06:47:24.924Z 16 320-339 183 183 183 312 312 312 F F F
201192024-02-08T06:47:24.924Z 17 340-359 178 178 178 303 303 303 F F F
201202024-02-08T06:47:24.924Z 18 360-379 185 185 185 317 317 317 F F F
201212024-02-08T06:47:24.924Z 19 380-399 185 185 185 317 317 317 F F F
201222024-02-08T06:47:24.924Z 20 400-419 182 182 182 311 311 311 F F F
201232024-02-08T06:47:24.924Z 21 420-439 185 185 185 317 317 317 F F F
201242024-02-08T06:47:24.924Z 22 440-459 185 185 185 317 317 317 F F F
201252024-02-08T06:47:24.925Z 23 460-479 182 182 182 311 311 311 F F F
201262024-02-08T06:47:24.925Z 24 480-499 182 182 182 311 311 311 F F F
201272024-02-08T06:47:24.925Z 25 500-519 182 182 182 311 311 311 F F F
201282024-02-08T06:47:24.925Z 26 520-539 184 184 184 315 315 315 F F F
201292024-02-08T06:47:24.925Z 27 540-559 182 182 182 311 311 311 F F F
201302024-02-08T06:47:24.925Z 28 560-579 184 184 184 315 315 315 F F F
201312024-02-08T06:47:24.925Z 29 580-599 184 184 184 315 315 315 F F F
201322024-02-08T06:47:24.925ZMax gen: 185, Max flush: 317
201332024-02-08T06:47:24.925Z
201342024-02-08T06:47:24.925ZBegin loop 93
201352024-02-08T06:47:24.925ZDownstairs to restart: 0
201362024-02-08T06:47:24.940Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.149438432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882}
201372024-02-08T06:47:24.940Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.149703132Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882}
201382024-02-08T06:47:24.941Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.149722661Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882}
201392024-02-08T06:47:24.946Z{"msg":"Crucible 71f48c5b-ab74-403d-84d6-dd937ecd9047 has session id: 84e15e47-9961-4997-87f3-4fa7adbf7eb2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.149755518Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201402024-02-08T06:47:24.946Z{"msg":"Upstairs opts: Upstairs UUID: 71f48c5b-ab74-403d-84d6-dd937ecd9047, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.149789036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201412024-02-08T06:47:24.946Z{"msg":"Crucible stats registered with UUID: 71f48c5b-ab74-403d-84d6-dd937ecd9047","v":0,"name":"crucible","level":30Crucible runtime is spawned
201422024-02-08T06:47:24.946Z,"time":"2024-02-08T06:47:24.149814624Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201432024-02-08T06:47:24.946Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.149920756Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882}
201442024-02-08T06:47:24.946Z{"msg":"71f48c5b-ab74-403d-84d6-dd937ecd9047 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.149992751Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201452024-02-08T06:47:24.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.150028578Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201462024-02-08T06:47:24.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.150060155Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201472024-02-08T06:47:24.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.150113072Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201482024-02-08T06:47:24.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.150148559Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201492024-02-08T06:47:24.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.150172847Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201502024-02-08T06:47:24.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.150209364Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201512024-02-08T06:47:24.946Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.15026946Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201522024-02-08T06:47:24.946Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.150326426Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201532024-02-08T06:47:24.947Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.150358163Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201542024-02-08T06:47:24.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.15039986Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201552024-02-08T06:47:24.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.150449326Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201562024-02-08T06:47:24.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.150476764Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201572024-02-08T06:47:24.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.151117176Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201582024-02-08T06:47:24.947Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.151154404Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201592024-02-08T06:47:24.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.151173642Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201602024-02-08T06:47:24.947Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.15119709Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201612024-02-08T06:47:24.947Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.151719851Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201622024-02-08T06:47:24.947Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.15186764Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201632024-02-08T06:47:24.947Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.152230173Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201642024-02-08T06:47:24.947Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.15226984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201652024-02-08T06:47:24.947Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.152286948Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201662024-02-08T06:47:24.947Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.152380821Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201672024-02-08T06:47:24.947Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.152421358Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201682024-02-08T06:47:24.948Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.152455436Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201692024-02-08T06:47:24.948Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.152487373Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201702024-02-08T06:47:24.948Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.152509982Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201712024-02-08T06:47:24.948Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.152998005Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201722024-02-08T06:47:24.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153447221Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201732024-02-08T06:47:24.952Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153491488Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201742024-02-08T06:47:24.952Z{"msg":"[0]R flush_numbers[0..12]: [309, 313, 313, 312, 317, 305, 305, 298, 312, 304, 316, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153508327Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201752024-02-08T06:47:24.952Z{"msg":"[0]R generation[0..12]: [182, 184, 184, 183, 185, 180, 180, 176, 183, 179, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153534775Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201762024-02-08T06:47:24.953Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153569532Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201772024-02-08T06:47:24.953Z{"msg":"[1]R flush_numbers[0..12]: [309, 313, 313, 312, 317, 305, 305, 298, 312, 304, 316, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.15360204Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201782024-02-08T06:47:24.953Z{"msg":"[1]R generation[0..12]: [182, 184, 184, 183, 185, 180, 180, 176, 183, 179, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153635797Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201792024-02-08T06:47:24.953Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153668595Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201802024-02-08T06:47:24.953Z{"msg":"Wait for a query_work_queue command to finish before sending IO
201812024-02-08T06:47:24.953Z[2]R flush_numbers[0..12]: [309, 313, 313, 312, 317, 305, 305, 298, 312, 304, 316, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153691113Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201822024-02-08T06:47:24.953Z{"msg":"[2]R generation[0..12]: [182, 184, 184, 183, 185, 180, 180, 176, 183, 179, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153720291Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201832024-02-08T06:47:24.953Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153738599Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201842024-02-08T06:47:24.953Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153767477Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201852024-02-08T06:47:24.953Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153785036Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201862024-02-08T06:47:24.953Z{"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153814174Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201872024-02-08T06:47:24.953Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153828923Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201882024-02-08T06:47:24.953Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.15385962Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201892024-02-08T06:47:24.953Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153873449Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201902024-02-08T06:47:24.953Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153904397Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201912024-02-08T06:47:24.954Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153920526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201922024-02-08T06:47:24.954Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153950274Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201932024-02-08T06:47:24.954Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.153964093Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201942024-02-08T06:47:24.954Z{"msg":"71f48c5b-ab74-403d-84d6-dd937ecd9047 is now active with session: 84e15e47-9961-4997-87f3-4fa7adbf7eb2","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.15399476Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201952024-02-08T06:47:24.954Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.154036297Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882,"session_id":"84e15e47-9961-4997-87f3-4fa7adbf7eb2"}
201962024-02-08T06:47:24.954Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:24.154067735Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882}
201972024-02-08T06:47:24.954ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
201982024-02-08T06:47:24.954ZLoading write count information from file "/tmp/test_repair_verify.data"
201992024-02-08T06:47:24.954ZRun Repair workload
202002024-02-08T06:47:24.954Z01/30 Write block 279 len 8192 data: 19 17
202012024-02-08T06:47:24.954Z02/30 Read block 294 len 24576
202022024-02-08T06:47:24.954Z03/30 Write block 87 len 24576 data: 15 14 13 14 17 20
202032024-02-08T06:47:24.954Z04/30 Read block 156 len 40960
202042024-02-08T06:47:24.954Z05/30 Read block 563 len 20480
202052024-02-08T06:47:24.954Z06/30 Write block 372 len 32768 data: 20 19 11 14 16 14 14 13
202062024-02-08T06:47:24.958Z07/30 Write block 0 len 32768 data: 4 5 5 7 6 7 8 9
202072024-02-08T06:47:24.958Z08/30 Read block 365 len 8192
202082024-02-08T06:47:24.958Z09/30 Read block 540 len 12288
202092024-02-08T06:47:24.958Z10/30 Read block 206 len 40960
202102024-02-08T06:47:24.958Z11/30 Read block 440 len 40960
202112024-02-08T06:47:24.958Z12/30 Read block 551 len 28672
202122024-02-08T06:47:24.958Z13/30 Write block 250 len 12288 data: 15 16 13
202132024-02-08T06:47:24.958Z14/30 Flush
202142024-02-08T06:47:24.962Z15/30 Flush
202152024-02-08T06:47:24.962Z16/30 Write block 315 len 28672 data: 9 10 9 12 14 19 16
202162024-02-08T06:47:24.964Z17/30 Write block 507 len 32768 data: 9 9 12 15 12 11 13 11
202172024-02-08T06:47:24.965Z18/30 Read block 104 len 24576
202182024-02-08T06:47:24.965Z19/30 Read block 512 len 28672
202192024-02-08T06:47:24.967Z20/30 Write block 351 len 20480 data: 14 14 13 13 13
202202024-02-08T06:47:24.967Z21/30 Write block 16 len 40960 data: 13 13 14 15 17 14 14 13 17 18
202212024-02-08T06:47:24.968Z22/30 Read block 583 len 12288
202222024-02-08T06:47:24.968Z23/30 Write block 246 len 28672 data: 15 12 12 16 16 17 14
202232024-02-08T06:47:24.970Z24/30 Write block 237 len 40960 data: 14 19 20 17 20 16 18 16 16 16
202242024-02-08T06:47:24.970Z25/30 Read block 511 len 28672
202252024-02-08T06:47:24.970Z26/30 Read block 6 len 16384
202262024-02-08T06:47:24.973Z27/30 Read block 45 len 8192
202272024-02-08T06:47:24.973Z28/30 Read block 531 len 8192
202282024-02-08T06:47:24.973Z29/30 Write block 111 len 20480 data: 10 12 16 16 17
202292024-02-08T06:47:24.973Z30/30 Read block 12 len 36864
202302024-02-08T06:47:24.977Z----------------------------------------------------------------
202312024-02-08T06:47:24.977Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
202322024-02-08T06:47:24.977ZStates: Active Active Active
202332024-02-08T06:47:24.977ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
202342024-02-08T06:47:24.977Z 1 Acked 1000 Write 2 Done Done Done false
202352024-02-08T06:47:24.977Z 2 Acked 1001 Read 6 Done Done Done false
202362024-02-08T06:47:24.977Z 3 Acked 1002 Write 6 Sent Done Done false
202372024-02-08T06:47:24.977Z 4 Acked 1003 Read 10 Sent Done Done false
202382024-02-08T06:47:24.978Z 5 Acked 1004 Read 5 Sent Done Done false
202392024-02-08T06:47:24.978Z 6 Acked 1005 Write 8 Sent Done Done false
202402024-02-08T06:47:24.978Z 7 Acked 1006 Write 8 Sent Done Done false
202412024-02-08T06:47:24.978Z 8 Acked 1007 Read 2 Sent Done Done false
202422024-02-08T06:47:24.978Z 9 Acked 1008 Read 3 Sent Done Done false
202432024-02-08T06:47:24.978Z 10 Acked 1009 Read 10 Sent Done Done false
202442024-02-08T06:47:24.978Z 11 Acked 1010 Read 10 Sent Done Done false
202452024-02-08T06:47:24.978Z 12 Acked 1011 Read 7 Sent Done Done false
202462024-02-08T06:47:24.978Z 13 Acked 1012 Write 3 Sent Done Done false
202472024-02-08T06:47:24.978Z 14 Acked 1013 Flush 0 Sent Done Done false
202482024-02-08T06:47:24.978Z 15 Acked 1014 Flush 0 Sent Done Done false
202492024-02-08T06:47:24.978Z 16 Acked 1015 Write 7 Sent Done Done false
202502024-02-08T06:47:24.978Z 17 Acked 1016 Write 8 Sent Done Done false
202512024-02-08T06:47:24.978Z 18 Acked 1017 Read 6 Sent Done Done false
202522024-02-08T06:47:24.978Z 19 Acked 1018 Read 7 Sent Done Done false
202532024-02-08T06:47:24.978Z 20 Acked 1019 Write 5 Sent Done Done false
202542024-02-08T06:47:24.979Z 21 Acked 1020 Write 10 Sent Done Done false
202552024-02-08T06:47:24.979Z 22 Acked 1021 Read 3 Sent Done Done false
202562024-02-08T06:47:24.979Z 23 Acked 1022 Write 7 Sent Done Done false
202572024-02-08T06:47:24.979Z 24 Acked 1023 Write 10 Sent Done Done false
202582024-02-08T06:47:24.979Z 25 Acked 1024 Read 7 Sent Done Done false
202592024-02-08T06:47:24.979Z 26 Acked 1025 Read 4 Sent Done Done false
202602024-02-08T06:47:24.979Z 27 Acked 1026 Read 2 Sent Done Done false
202612024-02-08T06:47:24.979Z 28 Acked 1027 Read 2 Sent Done Done false
202622024-02-08T06:47:24.979Z 29 Acked 1028 Write 5 Sent Done Done false
202632024-02-08T06:47:24.979Z 30 Acked 1029 Read 9 Sent Done Done false
202642024-02-08T06:47:24.979Z STATES DS:0 DS:1 DS:2 TOTAL
202652024-02-08T06:47:24.979Z New 0 0 0 0
202662024-02-08T06:47:24.979Z Sent 28 0 0 28
202672024-02-08T06:47:24.979Z Done 2 30 30 62
202682024-02-08T06:47:24.980Z Skipped 0 0 0 0
202692024-02-08T06:47:24.980Z Error 0 0 0 0
202702024-02-08T06:47:24.980ZLast Flush: 0 1014 1014
202712024-02-08T06:47:24.980ZDownstairs last five completed:
202722024-02-08T06:47:24.980ZUpstairs last five completed: 30 29 28 27 26
202732024-02-08T06:47:24.980Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:24.183816261Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1882}
202742024-02-08T06:47:24.980ZWrote out file "/tmp/test_repair_verify.data"
202752024-02-08T06:47:24.983Z
202762024-02-08T06:47:24.984ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202772024-02-08T06:47:24.999ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202782024-02-08T06:47:24.999Z 0 000-019 182 186 186 309 318 318 F T T <---
202792024-02-08T06:47:24.999Z 1 020-039 184 184 184 313 313 313 F T T <---
202802024-02-08T06:47:24.999Z 2 040-059 184 184 184 313 313 313 F F F
202812024-02-08T06:47:24.999Z 3 060-079 183 183 183 312 312 312 F F F
202822024-02-08T06:47:25.000Z 4 080-099 185 186 186 317 318 318 F F F <---
202832024-02-08T06:47:25.003Z 5 100-119 180 180 180 305 305 305 F T T <---
202842024-02-08T06:47:25.003Z 6 120-139 180 180 180 305 305 305 F F F
202852024-02-08T06:47:25.003Z 7 140-159 176 176 176 298 298 298 F F F
202862024-02-08T06:47:25.003Z 8 160-179 183 183 183 312 312 312 F F F
202872024-02-08T06:47:25.003Z 9 180-199 179 179 179 304 304 304 F F F
202882024-02-08T06:47:25.003Z 10 200-219 184 184 184 316 316 316 F F F
202892024-02-08T06:47:25.003Z 11 220-239 184 184 184 313 313 313 F T T <---
202902024-02-08T06:47:25.003Z 12 240-259 182 186 186 309 318 318 F T T <---
202912024-02-08T06:47:25.004Z 13 260-279 183 186 186 312 318 318 T F F <---
202922024-02-08T06:47:25.004Z 14 280-299 179 186 186 304 318 318 T F F <---
202932024-02-08T06:47:25.004Z 15 300-319 183 183 183 312 312 312 F T T <---
202942024-02-08T06:47:25.004Z 16 320-339 183 183 183 312 312 312 F T T <---
202952024-02-08T06:47:25.004Z 17 340-359 178 178 178 303 303 303 F T T <---
202962024-02-08T06:47:25.004Z 18 360-379 185 186 186 317 318 318 F F F <---
202972024-02-08T06:47:25.004Z 19 380-399 185 185 185 317 317 317 F F F
202982024-02-08T06:47:25.004Z 20 400-419 182 182 182 311 311 311 F F F
202992024-02-08T06:47:25.004Z 21 420-439 185 185 185 317 317 317 F F F
203002024-02-08T06:47:25.004Z 22 440-459 185 185 185 317 317 317 F F F
203012024-02-08T06:47:25.004Z 23 460-479 182 182 182 311 311 311 F F F
203022024-02-08T06:47:25.004Z 24 480-499 182 182 182 311 311 311 F F F
203032024-02-08T06:47:25.004Z 25 500-519 182 182 182 311 311 311 F T T <---
203042024-02-08T06:47:25.004Z 26 520-539 184 184 184 315 315 315 F F F
203052024-02-08T06:47:25.004Z 27 540-559 182 182 182 311 311 311 F F F
203062024-02-08T06:47:25.005Z 28 560-579 184 184 184 315 315 315 F F F
203072024-02-08T06:47:25.005Z 29 580-599 184 184 184 315 315 315 F F F
203082024-02-08T06:47:25.005ZMax gen: 186, Max flush: 318
203092024-02-08T06:47:25.005ZError: Difference in extent metadata found!
203102024-02-08T06:47:25.005ZA Difference in extent metadata is expected here
203112024-02-08T06:47:25.005ZOn loop 93
203122024-02-08T06:47:25.005Z
203132024-02-08T06:47:25.008ZVerifying data now
203142024-02-08T06:47:30.660ZNo change after verify
203152024-02-08T06:47:30.660ZLoop: 93 Downstairs dump after verify (and repair):
203162024-02-08T06:47:30.675ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203172024-02-08T06:47:30.675Z 0 000-019 187 187 187 319 319 319 F F F
203182024-02-08T06:47:30.675Z 1 020-039 187 187 187 319 319 319 F F F
203192024-02-08T06:47:30.675Z 2 040-059 184 184 184 313 313 313 F F F
203202024-02-08T06:47:30.675Z 3 060-079 183 183 183 312 312 312 F F F
203212024-02-08T06:47:30.675Z 4 080-099 186 186 186 318 318 318 F F F
203222024-02-08T06:47:30.675Z 5 100-119 187 187 187 319 319 319 F F F
203232024-02-08T06:47:30.675Z 6 120-139 180 180 180 305 305 305 F F F
203242024-02-08T06:47:30.675Z 7 140-159 176 176 176 298 298 298 F F F
203252024-02-08T06:47:30.675Z 8 160-179 183 183 183 312 312 312 F F F
203262024-02-08T06:47:30.675Z 9 180-199 179 179 179 304 304 304 F F F
203272024-02-08T06:47:30.678Z 10 200-219 184 184 184 316 316 316 F F F
203282024-02-08T06:47:30.678Z 11 220-239 187 187 187 319 319 319 F F F
203292024-02-08T06:47:30.679Z 12 240-259 187 187 187 319 319 319 F F F
203302024-02-08T06:47:30.679Z 13 260-279 186 186 186 318 318 318 F F F
203312024-02-08T06:47:30.679Z 14 280-299 186 186 186 318 318 318 F F F
203322024-02-08T06:47:30.679Z 15 300-319 187 187 187 319 319 319 F F F
203332024-02-08T06:47:30.679Z 16 320-339 187 187 187 319 319 319 F F F
203342024-02-08T06:47:30.679Z 17 340-359 187 187 187 319 319 319 F F F
203352024-02-08T06:47:30.679Z 18 360-379 186 186 186 318 318 318 F F F
203362024-02-08T06:47:30.679Z 19 380-399 185 185 185 317 317 317 F F F
203372024-02-08T06:47:30.679Z 20 400-419 182 182 182 311 311 311 F F F
203382024-02-08T06:47:30.679Z 21 420-439 185 185 185 317 317 317 F F F
203392024-02-08T06:47:30.679Z 22 440-459 185 185 185 317 317 317 F F F
203402024-02-08T06:47:30.679Z 23 460-479 182 182 182 311 311 311 F F F
203412024-02-08T06:47:30.679Z 24 480-499 182 182 182 311 311 311 F F F
203422024-02-08T06:47:30.679Z 25 500-519 187 187 187 319 319 319 F F F
203432024-02-08T06:47:30.679Z 26 520-539 184 184 184 315 315 315 F F F
203442024-02-08T06:47:30.680Z 27 540-559 182 182 182 311 311 311 F F F
203452024-02-08T06:47:30.680Z 28 560-579 184 184 184 315 315 315 F F F
203462024-02-08T06:47:30.680Z 29 580-599 184 184 184 315 315 315 F F F
203472024-02-08T06:47:30.680ZMax gen: 187, Max flush: 319
203482024-02-08T06:47:30.680Z
203492024-02-08T06:47:30.680ZBegin loop 94
203502024-02-08T06:47:30.680ZDownstairs to restart: 0
203512024-02-08T06:47:30.695Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.903401646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890Crucible runtime is spawned
203522024-02-08T06:47:30.695Z}
203532024-02-08T06:47:30.702Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.903724792Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890}
203542024-02-08T06:47:30.702Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.903775238Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890}
203552024-02-08T06:47:30.702Z{"msg":"Crucible 133e4d39-3d15-4c53-be59-d7022eecbcaa has session id: 0029c890-cf33-4fff-8c17-8d420a1a4b60","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.903814695Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203562024-02-08T06:47:30.702Z{"msg":"Upstairs opts: Upstairs UUID: 133e4d39-3d15-4c53-be59-d7022eecbcaa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.903868971Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203572024-02-08T06:47:30.702Z{"msg":"Crucible stats registered with UUID: 133e4d39-3d15-4c53-be59-d7022eecbcaa","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.903913487Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203582024-02-08T06:47:30.702Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.903955414Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890}
203592024-02-08T06:47:30.702Z{"msg":"133e4d39-3d15-4c53-be59-d7022eecbcaa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.903996241Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203602024-02-08T06:47:30.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904038008Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203612024-02-08T06:47:30.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904078855Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203622024-02-08T06:47:30.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904137001Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203632024-02-08T06:47:30.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904183387Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203642024-02-08T06:47:30.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904231094Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203652024-02-08T06:47:30.703Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904293399Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203662024-02-08T06:47:30.703Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904332076Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"io task","client":"1","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203672024-02-08T06:47:30.703Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904362524Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"io task","client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203682024-02-08T06:47:30.703Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904390402Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"io task","client":"2","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203692024-02-08T06:47:30.703Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:29.904420909Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"io task","client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203702024-02-08T06:47:30.703Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:29.904448937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"io task","client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203712024-02-08T06:47:30.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904472756Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"io task","client":"1","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203722024-02-08T06:47:30.703Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:29.904498514Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203732024-02-08T06:47:30.703Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904529111Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203742024-02-08T06:47:30.703Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.90454935Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203752024-02-08T06:47:30.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904576118Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"io task","client":"2","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203762024-02-08T06:47:30.703Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.904602806Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"io task","client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203772024-02-08T06:47:30.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.905028464Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203782024-02-08T06:47:30.703Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.905065491Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203792024-02-08T06:47:30.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.905083979Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203802024-02-08T06:47:30.704Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.905129866Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203812024-02-08T06:47:30.704Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.90561474Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203822024-02-08T06:47:30.704Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.905657147Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203832024-02-08T06:47:30.704Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.906196386Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203842024-02-08T06:47:30.704Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.906231833Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203852024-02-08T06:47:30.704Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.906247032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203862024-02-08T06:47:30.704Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.906267801Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203872024-02-08T06:47:30.704Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.90628168Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203882024-02-08T06:47:30.704Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:29.906301818Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203892024-02-08T06:47:31.701Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.905977944Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"io task","client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203902024-02-08T06:47:31.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.906276482Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"io task","client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203912024-02-08T06:47:31.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.907028855Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203922024-02-08T06:47:31.724Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.907063903Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203932024-02-08T06:47:31.724Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.90750789Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203942024-02-08T06:47:31.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.90803078Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203952024-02-08T06:47:31.725Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908073987Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203962024-02-08T06:47:31.725Z{"msg":"[0]R flush_numbers[0..12]: [319, 319, 313, 312, 318, 319, 305, 298, 312, 304, 316, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908095855Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203972024-02-08T06:47:31.725Z{"msg":"[0]R generation[0..12]: [187, 187, 184, 183, 186, 187, 180, 176, 183, 179, 184, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908122953Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203982024-02-08T06:47:31.726Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908141352Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
203992024-02-08T06:47:31.726Z{"msg":"[1]R flush_numbers[0..12]: [319, 319, 313, 312, 318, 319, 305, 298, 312, 304, 316, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.90816724Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204002024-02-08T06:47:31.726Z{"msg":"[1]R generation[0..12]: [187, 187, 184, 183, 186, 187, 180, 176, 183, 179, 184, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908191278Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204012024-02-08T06:47:31.726ZWait for a query_work_queue command to finish before sending IO
204022024-02-08T06:47:31.727Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908242794Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204032024-02-08T06:47:31.727Z{"msg":"[2]R flush_numbers[0..12]: [319, 319, 313, 312, 318, 319, 305, 298, 312, 304, 316, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908289211Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204042024-02-08T06:47:31.727Z{"msg":"[2]R generation[0..12]: [187, 187, 184, 183, 186, 187, 180, 176, 183, 179, 184, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908336067Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204052024-02-08T06:47:31.728Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908384914Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204062024-02-08T06:47:31.728Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.90842942Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204072024-02-08T06:47:31.728Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908466018Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204082024-02-08T06:47:31.728Z{"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908514034Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204092024-02-08T06:47:31.728Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908542272Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204102024-02-08T06:47:31.729Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908581679Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204112024-02-08T06:47:31.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908603047Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204122024-02-08T06:47:31.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908635975Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204132024-02-08T06:47:31.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908666263Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204142024-02-08T06:47:31.730Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.90869862Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204152024-02-08T06:47:31.730Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908717969Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204162024-02-08T06:47:31.730Z{"msg":"133e4d39-3d15-4c53-be59-d7022eecbcaa is now active with session: 0029c890-cf33-4fff-8c17-8d420a1a4b60","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908732078Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204172024-02-08T06:47:31.730Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908751876Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204182024-02-08T06:47:31.731Z0029c890-cf33-4fff-8c17-8d420a1a4b60"}
204192024-02-08T06:47:31.731Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:30.908786023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890}
204202024-02-08T06:47:31.732ZLoading write count information from file "/tmp/test_repair_verify.data"
204212024-02-08T06:47:31.732ZRun Repair workload
204222024-02-08T06:47:31.732Z01/30 Write block 282 len 36864 data: 13 12 14 13 10 11 13 12 16
204232024-02-08T06:47:31.733Z02/30 Read block 66 len 24576
204242024-02-08T06:47:31.733Z03/30 Write block 507 len 24576 data: 10 10 13 16 13 12
204252024-02-08T06:47:31.733Z04/30 Read block 478 len 20480
204262024-02-08T06:47:31.751Z05/30 Flush
204272024-02-08T06:47:31.751Z06/30 Write block 479 len 28672 data: 11 8 9 10 9 14 15
204282024-02-08T06:47:31.752Z07/30 Flush
204292024-02-08T06:47:31.752Z08/30 Write block 352 len 28672 data: 15 14 14 14 11 13 14
204302024-02-08T06:47:31.752Z09/30 Write block 284 len 28672 data: 15 14 11 12 14 13 17
204312024-02-08T06:47:31.752Z10/30 Read block 419 len 36864
204322024-02-08T06:47:31.752Z11/30 Read block 328 len 8192
204332024-02-08T06:47:31.753Z12/30 Flush
204342024-02-08T06:47:31.753Z13/30 Write block 253 len 32768 data: 9 11 14 12 11 13 16 16
204352024-02-08T06:47:31.753Z14/30 Read block 201 len 24576
204362024-02-08T06:47:31.753Z15/30 Flush
204372024-02-08T06:47:31.753Z16/30 Write block 540 len 24576 data: 15 14 11 14 14 15
204382024-02-08T06:47:31.754Z17/30 Write block 333 len 32768 data: 18 21 21 21 18 20 21 19
204392024-02-08T06:47:31.754Z18/30 Write block 436 len 24576 data: 15 19 18 15 16 16
204402024-02-08T06:47:31.754Z19/30 Read block 318 len 8192
204412024-02-08T06:47:31.754Z20/30 Read block 91 len 24576
204422024-02-08T06:47:31.754Z21/30 Read block 20 len 32768
204432024-02-08T06:47:31.754Z22/30 Write block 537 len 40960 data: 16 16 16 16 15 12 15 15 16 19
204442024-02-08T06:47:31.755Z23/30 Read block 539 len 24576
204452024-02-08T06:47:31.755Z24/30 Read block 323 len 8192
204462024-02-08T06:47:31.755Z25/30 Write block 83 len 12288 data: 19 18 17
204472024-02-08T06:47:31.755Z26/30 Flush
204482024-02-08T06:47:31.755Z27/30 Write block 216 len 4096 data: 20
204492024-02-08T06:47:31.756Z28/30 Read block 250 len 4096
204502024-02-08T06:47:31.756Z29/30 Read block 89 len 40960
204512024-02-08T06:47:31.756Z30/30 Read block 467 len 40960
204522024-02-08T06:47:31.756Z----------------------------------------------------------------
204532024-02-08T06:47:31.757Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
204542024-02-08T06:47:31.757ZStates: Active Active Active
204552024-02-08T06:47:31.757ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204562024-02-08T06:47:31.757Z 1 Acked 1000 Write 9 Done Done Done false
204572024-02-08T06:47:31.757Z 2 Acked 1001 Read 6 Done Done Done false
204582024-02-08T06:47:31.758Z 3 Acked 1002 Write 6 Done Done Done false
204592024-02-08T06:47:31.758Z 4 Acked 1003 Read 5 Sent Done Done false
204602024-02-08T06:47:31.758Z 5 Acked 1004 Flush 0 Sent Done Done false
204612024-02-08T06:47:31.758Z 6 Acked 1005 Write 7 Sent Done Done false
204622024-02-08T06:47:31.758Z 7 Acked 1006 Flush 0 Sent Done Done false
204632024-02-08T06:47:31.759Z 8 Acked 1007 Write 7 Sent Done Done false
204642024-02-08T06:47:31.759Z 9 Acked 1008 Write 7 Sent Done Done false
204652024-02-08T06:47:31.759Z 10 Acked 1009 Read 9 Sent Done Done false
204662024-02-08T06:47:31.759Z 11 Acked 1010 Read 2 Sent Done Done false
204672024-02-08T06:47:31.759Z 12 Acked 1011 Flush 0 Sent Done Done false
204682024-02-08T06:47:31.760Z 13 Acked 1012 Write 8 Sent Done Done false
204692024-02-08T06:47:31.760Z 14 Acked 1013 Read 6 Sent Done Done false
204702024-02-08T06:47:31.760Z 15 Acked 1014 Flush 0 Sent Done Done false
204712024-02-08T06:47:31.760Z 16 Acked 1015 Write 6 Sent Done Done false
204722024-02-08T06:47:31.760Z 17 Acked 1016 Write 8 Sent Done Done false
204732024-02-08T06:47:31.761Z 18 Acked 1017 Write 6 Sent Done Done false
204742024-02-08T06:47:31.761Z 19 Acked 1018 Read 2 Sent Done Done false
204752024-02-08T06:47:31.761Z 20 Acked 1019 Read 6 Sent Done Done false
204762024-02-08T06:47:31.761Z 21 Acked 1020 Read 8 Sent Done Done false
204772024-02-08T06:47:31.762Z 22 Acked 1021 Write 10 Sent Done Done false
204782024-02-08T06:47:31.762Z 23 Acked 1022 Read 6 Sent Done Done false
204792024-02-08T06:47:31.762Z 24 Acked 1023 Read 2 Sent Done Done false
204802024-02-08T06:47:31.762Z 25 Acked 1024 Write 3 Sent Done Done false
204812024-02-08T06:47:31.762Z 26 Acked 1025 Flush 0 Sent Done Done false
204822024-02-08T06:47:31.763Z 27 Acked 1026 Write 1 Sent Done Done false
204832024-02-08T06:47:31.763Z 28 Acked 1027 Read 1 Sent Done Done false
204842024-02-08T06:47:31.763Z 29 Acked 1028 Read 10 Sent Done Done false
204852024-02-08T06:47:31.763Z 30 Acked 1029 Read 10 Sent Done Sent false
204862024-02-08T06:47:31.763Z STATES DS:0 DS:1 DS:2 TOTAL
204872024-02-08T06:47:31.764Z New 0 0 0 0
204882024-02-08T06:47:31.764Z Sent 27 0 1 28
204892024-02-08T06:47:31.764Z Done 3 30 29 62
204902024-02-08T06:47:31.764Z Skipped 0 0 0 0
204912024-02-08T06:47:31.765Z Error 0 0 0 0
204922024-02-08T06:47:31.765ZLast Flush: 0 1025 1025
204932024-02-08T06:47:31.765ZDownstairs last five completed:
204942024-02-08T06:47:31.765ZUpstairs last five completed: 30 29 28 27 26
204952024-02-08T06:47:31.765Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:30.945161832Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1890}
204962024-02-08T06:47:31.766ZWrote out file "/tmp/test_repair_verify.data"
204972024-02-08T06:47:31.766Z
204982024-02-08T06:47:32.302ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
204992024-02-08T06:47:32.302ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205002024-02-08T06:47:32.302Z 0 000-019 187 187 187 319 319 319 F F F
205012024-02-08T06:47:32.302Z 1 020-039 187 187 187 319 319 319 F F F
205022024-02-08T06:47:32.302Z 2 040-059 184 184 184 313 313 313 F F F
205032024-02-08T06:47:32.302Z 3 060-079 183 183 183 312 312 312 F F F
205042024-02-08T06:47:32.302Z 4 080-099 186 188 188 318 324 324 F F F <---
205052024-02-08T06:47:32.302Z 5 100-119 187 187 187 319 319 319 F F F
205062024-02-08T06:47:32.302Z 6 120-139 180 180 180 305 305 305 F F F
205072024-02-08T06:47:32.302Z 7 140-159 176 176 176 298 298 298 F F F
205082024-02-08T06:47:32.302Z 8 160-179 183 183 183 312 312 312 F F F
205092024-02-08T06:47:32.303Z 9 180-199 179 179 179 304 304 304 F F F
205102024-02-08T06:47:32.303Z 10 200-219 184 184 184 316 316 316 F T T <---
205112024-02-08T06:47:32.303Z 11 220-239 187 187 187 319 319 319 F F F
205122024-02-08T06:47:32.303Z 12 240-259 187 188 188 319 323 323 F F F <---
205132024-02-08T06:47:32.303Z 13 260-279 186 188 188 318 323 323 F F F <---
205142024-02-08T06:47:32.303Z 14 280-299 186 188 188 318 322 322 T F F <---
205152024-02-08T06:47:32.303Z 15 300-319 187 187 187 319 319 319 F F F
205162024-02-08T06:47:32.303Z 16 320-339 187 188 188 319 324 324 F F F <---
205172024-02-08T06:47:32.303Z 17 340-359 187 188 188 319 324 324 F F F <---
205182024-02-08T06:47:32.303Z 18 360-379 186 186 186 318 318 318 F F F
205192024-02-08T06:47:32.303Z 19 380-399 185 185 185 317 317 317 F F F
205202024-02-08T06:47:32.303Z 20 400-419 182 182 182 311 311 311 F F F
205212024-02-08T06:47:32.303Z 21 420-439 185 188 188 317 324 324 F F F <---
205222024-02-08T06:47:32.303Z 22 440-459 185 188 188 317 324 324 F F F <---
205232024-02-08T06:47:32.303Z 23 460-479 182 188 188 311 321 321 F F F <---
205242024-02-08T06:47:32.304Z 24 480-499 182 188 188 311 321 321 F F F <---
205252024-02-08T06:47:32.304Z 25 500-519 187 188 188 319 320 320 T F F <---
205262024-02-08T06:47:32.304Z 26 520-539 184 188 188 315 324 324 F F F <---
205272024-02-08T06:47:32.304Z 27 540-559 182 188 188 311 324 324 F F F <---
205282024-02-08T06:47:32.304Z 28 560-579 184 184 184 315 315 315 F F F
205292024-02-08T06:47:32.304Z 29 580-599 184 184 184 315 315 315 F F F
205302024-02-08T06:47:32.304ZMax gen: 188, Max flush: 324
205312024-02-08T06:47:32.304ZError: Difference in extent metadata found!
205322024-02-08T06:47:32.304ZA Difference in extent metadata is expected here
205332024-02-08T06:47:32.304ZOn loop 94
205342024-02-08T06:47:32.304Z
205352024-02-08T06:47:32.304ZVerifying data now
205362024-02-08T06:47:37.444ZNo change after verify
205372024-02-08T06:47:37.444ZLoop: 94 Downstairs dump after verify (and repair):
205382024-02-08T06:47:37.459ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205392024-02-08T06:47:37.459Z 0 000-019 187 187 187 319 319 319 F F F
205402024-02-08T06:47:37.459Z 1 020-039 187 187 187 319 319 319 F F F
205412024-02-08T06:47:37.459Z 2 040-059 184 184 184 313 313 313 F F F
205422024-02-08T06:47:37.462Z 3 060-079 183 183 183 312 312 312 F F F
205432024-02-08T06:47:37.462Z 4 080-099 188 188 188 324 324 324 F F F
205442024-02-08T06:47:37.462Z 5 100-119 187 187 187 319 319 319 F F F
205452024-02-08T06:47:37.462Z 6 120-139 180 180 180 305 305 305 F F F
205462024-02-08T06:47:37.462Z 7 140-159 176 176 176 298 298 298 F F F
205472024-02-08T06:47:37.462Z 8 160-179 183 183 183 312 312 312 F F F
205482024-02-08T06:47:37.463Z 9 180-199 179 179 179 304 304 304 F F F
205492024-02-08T06:47:37.463Z 10 200-219 189 189 189 325 325 325 F F F
205502024-02-08T06:47:37.463Z 11 220-239 187 187 187 319 319 319 F F F
205512024-02-08T06:47:37.463Z 12 240-259 188 188 188 323 323 323 F F F
205522024-02-08T06:47:37.463Z 13 260-279 188 188 188 323 323 323 F F F
205532024-02-08T06:47:37.463Z 14 280-299 188 188 188 322 322 322 F F F
205542024-02-08T06:47:37.463Z 15 300-319 187 187 187 319 319 319 F F F
205552024-02-08T06:47:37.463Z 16 320-339 188 188 188 324 324 324 F F F
205562024-02-08T06:47:37.463Z 17 340-359 188 188 188 324 324 324 F F F
205572024-02-08T06:47:37.463Z 18 360-379 186 186 186 318 318 318 F F F
205582024-02-08T06:47:37.463Z 19 380-399 185 185 185 317 317 317 F F F
205592024-02-08T06:47:37.463Z 20 400-419 182 182 182 311 311 311 F F F
205602024-02-08T06:47:37.463Z 21 420-439 188 188 188 324 324 324 F F F
205612024-02-08T06:47:37.463Z 22 440-459 188 188 188 324 324 324 F F F
205622024-02-08T06:47:37.464Z 23 460-479 188 188 188 321 321 321 F F F
205632024-02-08T06:47:37.464Z 24 480-499 188 188 188 321 321 321 F F F
205642024-02-08T06:47:37.464Z 25 500-519 188 188 188 320 320 320 F F F
205652024-02-08T06:47:37.464Z 26 520-539 188 188 188 324 324 324 F F F
205662024-02-08T06:47:37.464Z 27 540-559 188 188 188 324 324 324 F F F
205672024-02-08T06:47:37.464Z 28 560-579 184 184 184 315 315 315 F F F
205682024-02-08T06:47:37.464Z 29 580-599 184 184 184 315 315 315 F F F
205692024-02-08T06:47:37.464ZMax gen: 189, Max flush: 325
205702024-02-08T06:47:37.464Z
205712024-02-08T06:47:37.464ZBegin loop 95
205722024-02-08T06:47:37.464ZDownstairs to restart: 0
205732024-02-08T06:47:37.482Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.687324493Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898}
205742024-02-08T06:47:37.483Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"Crucible runtime is spawned
205752024-02-08T06:47:37.489Zx86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.687705865Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898}
205762024-02-08T06:47:37.489Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.687741942Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898}
205772024-02-08T06:47:37.489Z{"msg":"Crucible 89d432e3-15c1-44a3-8f41-aab4633c3ac5 has session id: fc893e9d-ce74-4a28-baa7-5c753e22a1b8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.68776391Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205782024-02-08T06:47:37.489Z{"msg":"Upstairs opts: Upstairs UUID: 89d432e3-15c1-44a3-8f41-aab4633c3ac5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.687805937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205792024-02-08T06:47:37.490Z{"msg":"Crucible stats registered with UUID: 89d432e3-15c1-44a3-8f41-aab4633c3ac5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.687848814Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205802024-02-08T06:47:37.490Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.687870792Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898}
205812024-02-08T06:47:37.490Z{"msg":"89d432e3-15c1-44a3-8f41-aab4633c3ac5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.68790042Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205822024-02-08T06:47:37.490Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.687930338Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205832024-02-08T06:47:37.490Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.687957066Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205842024-02-08T06:47:37.490Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.687989484Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205852024-02-08T06:47:37.490Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.68803957Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205862024-02-08T06:47:37.490Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.688065798Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205872024-02-08T06:47:37.490Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.688089636Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205882024-02-08T06:47:37.490Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.688112314Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"io task","client":"0","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205892024-02-08T06:47:37.490Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.688145872Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"io task","client":"2","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205902024-02-08T06:47:37.490Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.688179389Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"io task","client":"1","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205912024-02-08T06:47:37.490Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.688203777Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"io task","client":"2","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205922024-02-08T06:47:37.490Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.688227475Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"io task","client":"1","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205932024-02-08T06:47:37.490Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.688267243Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"io task","client":"0","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205942024-02-08T06:47:37.491Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.689062773Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205952024-02-08T06:47:37.491Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.689092321Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205962024-02-08T06:47:37.491Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.689109249Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205972024-02-08T06:47:37.491Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.689172285Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205982024-02-08T06:47:37.491Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.689668377Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
205992024-02-08T06:47:37.491Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.689823156Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206002024-02-08T06:47:37.491Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.689873032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206012024-02-08T06:47:37.491Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.68990104Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206022024-02-08T06:47:37.491Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.690216476Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206032024-02-08T06:47:37.491Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.690252483Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206042024-02-08T06:47:37.491Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.690268352Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206052024-02-08T06:47:37.491Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.690449269Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206062024-02-08T06:47:37.491Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.690480986Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206072024-02-08T06:47:37.491Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.690496545Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206082024-02-08T06:47:37.491Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.690517484Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206092024-02-08T06:47:37.491Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.690945662Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206102024-02-08T06:47:37.492Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.690983359Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206112024-02-08T06:47:37.492Z{"msg":"[0]R flush_numbers[0..12]: [319, 319, 313, 312, 324, 319, 305, 298, 312, 304, 325, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691025016Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206122024-02-08T06:47:37.492Z{"msg":"[0]R generation[0..12]: [187, 187, 184, 183, 188, 187, 180, 176, 183, 179, 189, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691051314Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206132024-02-08T06:47:37.492Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691066622Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206142024-02-08T06:47:37.492Z{"msg":"[1]R flush_numbers[0..12]: [319, 319, 313, 312, 324, 319, 305, 298, 312, 304, 325, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691081311Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206152024-02-08T06:47:37.492Z{"msg":"[1]R generation[0..12]: [187, 187, 184, 183, 188, 187, 180, 176, 183, 179, 189, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.69109531Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206162024-02-08T06:47:37.492Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691116659Z","hostname":Wait for a query_work_queue command to finish before sending IO
206172024-02-08T06:47:37.492Z"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206182024-02-08T06:47:37.492Z{"msg":"[2]R flush_numbers[0..12]: [319, 319, 313, 312, 324, 319, 305, 298, 312, 304, 325, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691134557Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206192024-02-08T06:47:37.492Z{"msg":"[2]R generation[0..12]: [187, 187, 184, 183, 188, 187, 180, 176, 183, 179, 189, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691155906Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206202024-02-08T06:47:37.492Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691170315Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206212024-02-08T06:47:37.492Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691197783Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206222024-02-08T06:47:37.492Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.69123025Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206232024-02-08T06:47:37.492Z{"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691262758Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206242024-02-08T06:47:37.492Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691294015Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"":"downstairs","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206252024-02-08T06:47:37.493Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691318023Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206262024-02-08T06:47:37.493Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691343332Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"0","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206272024-02-08T06:47:37.493Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.69136719Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"1","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206282024-02-08T06:47:37.493Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691394788Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"client":"2","":"downstairs","session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206292024-02-08T06:47:37.493Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691418686Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206302024-02-08T06:47:37.493Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691444784Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206312024-02-08T06:47:37.493Z{"msg":"89d432e3-15c1-44a3-8f41-aab4633c3ac5 is now active with session: fc893e9d-ce74-4a28-baa7-5c753e22a1b8","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691468352Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206322024-02-08T06:47:37.493ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206332024-02-08T06:47:37.493Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.69149662Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898,"session_id":"fc893e9d-ce74-4a28-baa7-5c753e22a1b8"}
206342024-02-08T06:47:37.493Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:36.691518799Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898}
206352024-02-08T06:47:37.493ZLoading write count information from file "/tmp/test_repair_verify.data"
206362024-02-08T06:47:37.493ZRun Repair workload
206372024-02-08T06:47:37.493Z01/30 Flush
206382024-02-08T06:47:37.493Z02/30 Write block 492 len 4096 data: 16
206392024-02-08T06:47:37.493Z03/30 Read block 15 len 8192
206402024-02-08T06:47:37.497Z04/30 Read block 447 len 32768
206412024-02-08T06:47:37.497Z05/30 Write block 478 len 36864 data: 10 12 9 10 11 10 15 16 17
206422024-02-08T06:47:37.497Z06/30 Read block 104 len 20480
206432024-02-08T06:47:37.497Z07/30 Write block 555 len 4096 data: 22
206442024-02-08T06:47:37.497Z08/30 Write block 126 len 12288 data: 10 8 10
206452024-02-08T06:47:37.497Z09/30 Read block 52 len 24576
206462024-02-08T06:47:37.497Z10/30 Read block 334 len 16384
206472024-02-08T06:47:37.498Z11/30 Read block 316 len 20480
206482024-02-08T06:47:37.498Z12/30 Write block 337 len 16384 data: 19 21 22 20
206492024-02-08T06:47:37.498Z13/30 Write block 402 len 40960 data: 14 13 14 15 14 14 15 15 13 11
206502024-02-08T06:47:37.498Z14/30 Write block 43 len 16384 data: 6 7 5 9
206512024-02-08T06:47:37.498Z15/30 Read block 571 len 28672
206522024-02-08T06:47:37.498Z16/30 Read block 228 len 40960
206532024-02-08T06:47:37.501Z17/30 Read block 302 len 20480
206542024-02-08T06:47:37.501Z18/30 Write block 420 len 36864 data: 17 17 15 15 14 12 12 10 9
206552024-02-08T06:47:37.501Z19/30 Read block 348 len 20480
206562024-02-08T06:47:37.501Z20/30 Read block 358 len 36864
206572024-02-08T06:47:37.504Z21/30 Write block 489 len 4096 data: 16
206582024-02-08T06:47:37.504Z22/30 Read block 451 len 36864
206592024-02-08T06:47:37.504Z23/30 Write block 232 len 40960 data: 9 11 12 16 14 15 20 21 18 21
206602024-02-08T06:47:37.504Z24/30 Read block 299 len 12288
206612024-02-08T06:47:37.507Z25/30 Write block 539 len 16384 data: 17 17 16 13
206622024-02-08T06:47:37.507Z26/30 Read block 74 len 16384
206632024-02-08T06:47:37.507Z27/30 Read block 539 len 36864
206642024-02-08T06:47:37.511Z28/30 Read block 212 len 4096
206652024-02-08T06:47:37.511Z29/30 Read block 102 len 32768
206662024-02-08T06:47:37.511Z30/30 Write block 279 len 20480 data: 20 18 15 14 13
206672024-02-08T06:47:37.511Z----------------------------------------------------------------
206682024-02-08T06:47:37.511Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
206692024-02-08T06:47:37.511ZStates: Active Active Active
206702024-02-08T06:47:37.511ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206712024-02-08T06:47:37.511Z 1 Acked 1000 Flush 0 Sent Done Done false
206722024-02-08T06:47:37.511Z 2 Acked 1001 Write 1 Sent Done Done false
206732024-02-08T06:47:37.511Z 3 Acked 1002 Read 2 Sent Done Done false
206742024-02-08T06:47:37.511Z 4 Acked 1003 Read 8 Sent Done Done false
206752024-02-08T06:47:37.511Z 5 Acked 1004 Write 9 Sent Done Done false
206762024-02-08T06:47:37.511Z 6 Acked 1005 Read 5 Sent Done Done false
206772024-02-08T06:47:37.512Z 7 Acked 1006 Write 1 Sent Done Done false
206782024-02-08T06:47:37.512Z 8 Acked 1007 Write 3 Sent Done Done false
206792024-02-08T06:47:37.512Z 9 Acked 1008 Read 6 Sent Done Done false
206802024-02-08T06:47:37.512Z 10 Acked 1009 Read 4 Sent Done Done false
206812024-02-08T06:47:37.512Z 11 Acked 1010 Read 5 Sent Done Done false
206822024-02-08T06:47:37.512Z 12 Acked 1011 Write 4 Sent Done Done false
206832024-02-08T06:47:37.512Z 13 Acked 1012 Write 10 Sent Done Done false
206842024-02-08T06:47:37.512Z 14 Acked 1013 Write 4 Sent Done Done false
206852024-02-08T06:47:37.512Z 15 Acked 1014 Read 7 Sent Done Done false
206862024-02-08T06:47:37.512Z 16 Acked 1015 Read 10 Sent Done Done false
206872024-02-08T06:47:37.512Z 17 Acked 1016 Read 5 Sent Done Done false
206882024-02-08T06:47:37.512Z 18 Acked 1017 Write 9 Sent Done Done false
206892024-02-08T06:47:37.512Z 19 Acked 1018 Read 5 Sent Done Done false
206902024-02-08T06:47:37.512Z 20 Acked 1019 Read 9 Sent Done Done false
206912024-02-08T06:47:37.512Z 21 Acked 1020 Write 1 Sent Done Done false
206922024-02-08T06:47:37.512Z 22 Acked 1021 Read 9 Sent Done Done false
206932024-02-08T06:47:37.513Z 23 Acked 1022 Write 10 Sent Done Done false
206942024-02-08T06:47:37.513Z 24 Acked 1023 Read 3 Sent Done Done false
206952024-02-08T06:47:37.513Z 25 Acked 1024 Write 4 Sent Done Done false
206962024-02-08T06:47:37.513Z 26 Acked 1025 Read 4 Sent Done Done false
206972024-02-08T06:47:37.513Z 27 Acked 1026 Read 9 Sent Done Done false
206982024-02-08T06:47:37.513Z 28 Acked 1027 Read 1 Sent Done Done false
206992024-02-08T06:47:37.513Z 29 Acked 1028 Read 8 Sent Done Done false
207002024-02-08T06:47:37.513Z 30 Acked 1029 Write 5 Sent Sent Sent false
207012024-02-08T06:47:37.513Z STATES DS:0 DS:1 DS:2 TOTAL
207022024-02-08T06:47:37.513Z New 0 0 0 0
207032024-02-08T06:47:37.513Z Sent 30 1 1 32
207042024-02-08T06:47:37.513Z Done 0 29 29 58
207052024-02-08T06:47:37.513Z Skipped 0 0 0 0
207062024-02-08T06:47:37.513Z Error 0 0 0 0
207072024-02-08T06:47:37.513ZLast Flush: 0 1000 1000
207082024-02-08T06:47:37.514ZDownstairs last five completed:
207092024-02-08T06:47:37.514ZUpstairs last five completed: 30 29 28 27 26
207102024-02-08T06:47:37.514Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:36.718015749Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1898}
207112024-02-08T06:47:37.514ZWrote out file "/tmp/test_repair_verify.data"
207122024-02-08T06:47:37.517Z
207132024-02-08T06:47:37.517ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207142024-02-08T06:47:37.536ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207152024-02-08T06:47:37.536Z 0 000-019 187 187 187 319 319 319 F F F
207162024-02-08T06:47:37.537Z 1 020-039 187 187 187 319 319 319 F F F
207172024-02-08T06:47:37.537Z 2 040-059 184 184 184 313 313 313 F T T <---
207182024-02-08T06:47:37.537Z 3 060-079 183 183 183 312 312 312 F F F
207192024-02-08T06:47:37.537Z 4 080-099 188 188 188 324 324 324 F F F
207202024-02-08T06:47:37.537Z 5 100-119 187 187 187 319 319 319 F F F
207212024-02-08T06:47:37.537Z 6 120-139 180 180 180 305 305 305 F T T <---
207222024-02-08T06:47:37.537Z 7 140-159 176 176 176 298 298 298 F F F
207232024-02-08T06:47:37.544Z 8 160-179 183 183 183 312 312 312 F F F
207242024-02-08T06:47:37.544Z 9 180-199 179 179 179 304 304 304 F F F
207252024-02-08T06:47:37.544Z 10 200-219 189 189 189 325 325 325 F F F
207262024-02-08T06:47:37.544Z 11 220-239 187 187 187 319 319 319 F T T <---
207272024-02-08T06:47:37.544Z 12 240-259 188 188 188 323 323 323 F T T <---
207282024-02-08T06:47:37.545Z 13 260-279 188 188 188 323 323 323 F T T <---
207292024-02-08T06:47:37.545Z 14 280-299 188 188 188 322 322 322 F T T <---
207302024-02-08T06:47:37.545Z 15 300-319 187 187 187 319 319 319 F F F
207312024-02-08T06:47:37.545Z 16 320-339 188 188 188 324 324 324 F T T <---
207322024-02-08T06:47:37.545Z 17 340-359 188 188 188 324 324 324 F T T <---
207332024-02-08T06:47:37.545Z 18 360-379 186 186 186 318 318 318 F F F
207342024-02-08T06:47:37.545Z 19 380-399 185 185 185 317 317 317 F F F
207352024-02-08T06:47:37.545Z 20 400-419 182 182 182 311 311 311 F T T <---
207362024-02-08T06:47:37.545Z 21 420-439 188 188 188 324 324 324 F T T <---
207372024-02-08T06:47:37.545Z 22 440-459 188 188 188 324 324 324 F F F
207382024-02-08T06:47:37.545Z 23 460-479 188 188 188 321 321 321 F T T <---
207392024-02-08T06:47:37.545Z 24 480-499 188 188 188 321 321 321 F T T <---
207402024-02-08T06:47:37.546Z 25 500-519 188 188 188 320 320 320 F F F
207412024-02-08T06:47:37.546Z 26 520-539 188 188 188 324 324 324 F T T <---
207422024-02-08T06:47:37.546Z 27 540-559 188 188 188 324 324 324 F T T <---
207432024-02-08T06:47:37.546Z 28 560-579 184 184 184 315 315 315 F F F
207442024-02-08T06:47:37.546Z 29 580-599 184 184 184 315 315 315 F F F
207452024-02-08T06:47:37.546ZMax gen: 189, Max flush: 325
207462024-02-08T06:47:37.546ZError: Difference in extent metadata found!
207472024-02-08T06:47:37.546ZA Difference in extent metadata is expected here
207482024-02-08T06:47:37.546ZOn loop 95
207492024-02-08T06:47:37.546Z
207502024-02-08T06:47:37.556ZVerifying data now
207512024-02-08T06:47:42.252ZNo change after verify
207522024-02-08T06:47:42.252ZLoop: 95 Downstairs dump after verify (and repair):
207532024-02-08T06:47:42.265ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207542024-02-08T06:47:42.265Z 0 000-019 187 187 187 319 319 319 F F F
207552024-02-08T06:47:42.265Z 1 020-039 187 187 187 319 319 319 F F F
207562024-02-08T06:47:42.266Z 2 040-059 190 190 190 326 326 326 F F F
207572024-02-08T06:47:42.266Z 3 060-079 183 183 183 312 312 312 F F F
207582024-02-08T06:47:42.266Z 4 080-099 188 188 188 324 324 324 F F F
207592024-02-08T06:47:42.266Z 5 100-119 187 187 187 319 319 319 F F F
207602024-02-08T06:47:42.266Z 6 120-139 190 190 190 326 326 326 F F F
207612024-02-08T06:47:42.266Z 7 140-159 176 176 176 298 298 298 F F F
207622024-02-08T06:47:42.270Z 8 160-179 183 183 183 312 312 312 F F F
207632024-02-08T06:47:42.270Z 9 180-199 179 179 179 304 304 304 F F F
207642024-02-08T06:47:42.270Z 10 200-219 189 189 189 325 325 325 F F F
207652024-02-08T06:47:42.270Z 11 220-239 190 190 190 326 326 326 F F F
207662024-02-08T06:47:42.270Z 12 240-259 190 190 190 326 326 326 F F F
207672024-02-08T06:47:42.270Z 13 260-279 190 190 190 326 326 326 F F F
207682024-02-08T06:47:42.270Z 14 280-299 190 190 190 326 326 326 F F F
207692024-02-08T06:47:42.270Z 15 300-319 187 187 187 319 319 319 F F F
207702024-02-08T06:47:42.270Z 16 320-339 190 190 190 326 326 326 F F F
207712024-02-08T06:47:42.270Z 17 340-359 190 190 190 326 326 326 F F F
207722024-02-08T06:47:42.270Z 18 360-379 186 186 186 318 318 318 F F F
207732024-02-08T06:47:42.271Z 19 380-399 185 185 185 317 317 317 F F F
207742024-02-08T06:47:42.271Z 20 400-419 190 190 190 326 326 326 F F F
207752024-02-08T06:47:42.271Z 21 420-439 190 190 190 326 326 326 F F F
207762024-02-08T06:47:42.271Z 22 440-459 188 188 188 324 324 324 F F F
207772024-02-08T06:47:42.271Z 23 460-479 190 190 190 326 326 326 F F F
207782024-02-08T06:47:42.271Z 24 480-499 190 190 190 326 326 326 F F F
207792024-02-08T06:47:42.271Z 25 500-519 188 188 188 320 320 320 F F F
207802024-02-08T06:47:42.271Z 26 520-539 190 190 190 326 326 326 F F F
207812024-02-08T06:47:42.271Z 27 540-559 190 190 190 326 326 326 F F F
207822024-02-08T06:47:42.271Z 28 560-579 184 184 184 315 315 315 F F F
207832024-02-08T06:47:42.271Z 29 580-599 184 184 184 315 315 315 F F F
207842024-02-08T06:47:42.271ZMax gen: 190, Max flush: 326
207852024-02-08T06:47:42.271Z
207862024-02-08T06:47:42.271ZBegin loop 96
207872024-02-08T06:47:42.271ZDownstairs to restart: 0
207882024-02-08T06:47:42.286Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.49251805Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906}
207892024-02-08T06:47:42.286Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
207902024-02-08T06:47:42.286Z,"time":"2024-02-08T06:47:41.49278267Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906}
207912024-02-08T06:47:42.293Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.492827087Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906}
207922024-02-08T06:47:42.293Z{"msg":"Crucible 280a6cd3-3bb4-47db-9ed7-29423d1d83c6 has session id: d276a78c-5302-4a1f-a0b9-1ca0f9d81f13","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.492874013Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
207932024-02-08T06:47:42.294Z{"msg":"Upstairs opts: Upstairs UUID: 280a6cd3-3bb4-47db-9ed7-29423d1d83c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.492908281Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
207942024-02-08T06:47:42.294Z{"msg":"Crucible stats registered with UUID: 280a6cd3-3bb4-47db-9ed7-29423d1d83c6","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.492937898Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
207952024-02-08T06:47:42.294Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.492952697Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906}
207962024-02-08T06:47:42.294Z{"msg":"280a6cd3-3bb4-47db-9ed7-29423d1d83c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.492984705Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
207972024-02-08T06:47:42.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.493017332Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
207982024-02-08T06:47:42.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.49304853Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
207992024-02-08T06:47:42.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.493073978Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208002024-02-08T06:47:42.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.493098826Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208012024-02-08T06:47:42.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.493120945Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208022024-02-08T06:47:42.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.493154682Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208032024-02-08T06:47:42.294Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.493188589Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"io task","client":"0","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208042024-02-08T06:47:42.294Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.493212787Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"io task","client":"2","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208052024-02-08T06:47:42.294Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.493246355Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"io task","client":"1","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208062024-02-08T06:47:42.294Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.493278553Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"io task","client":"1","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208072024-02-08T06:47:42.294Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.493387725Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"io task","client":"0","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208082024-02-08T06:47:42.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.4934502Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"io task","client":"2","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208092024-02-08T06:47:42.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.494068563Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208102024-02-08T06:47:42.295Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.49410672Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208112024-02-08T06:47:42.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.494123919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208122024-02-08T06:47:42.295Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.494157567Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208132024-02-08T06:47:42.295Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.494693526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208142024-02-08T06:47:42.295Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.494823007Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208152024-02-08T06:47:42.295Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.495250394Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208162024-02-08T06:47:42.295Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.495290391Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208172024-02-08T06:47:42.295Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.49530785Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208182024-02-08T06:47:42.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.495398274Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208192024-02-08T06:47:42.295Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.495424872Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208202024-02-08T06:47:42.295Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.49543926Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208212024-02-08T06:47:42.295Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.495462819Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208222024-02-08T06:47:42.296Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.495498226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208232024-02-08T06:47:42.296Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496016677Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208242024-02-08T06:47:42.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496437136Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208252024-02-08T06:47:42.296Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496470943Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208262024-02-08T06:47:42.296Z{"msg":"[0]R flush_numbers[0..12]: [319, 319, 326, 312, 324, 319, 326, 298, 312, 304, 325, 326]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.4965028Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208272024-02-08T06:47:42.296Z{"msg":"[0]R generation[0..12]: [187, 187, 190, 183, 188, 187, 190, 176, 183, 179, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496525189Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208282024-02-08T06:47:42.302Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496561646Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208292024-02-08T06:47:42.302Z{"msg":"[1]R flush_numbers[0..12]: [319, 319, 326, 312, 324, 319, 326, 298, 312, 304, 325, 326]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496600513Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208302024-02-08T06:47:42.303Z{Wait for a query_work_queue command to finish before sending IO
208312024-02-08T06:47:42.303Z"msg":"[1]R generation[0..12]: [187, 187, 190, 183, 188, 187, 190, 176, 183, 179, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.49663867Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208322024-02-08T06:47:42.303Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496671428Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208332024-02-08T06:47:42.303Z{"msg":"[2]R flush_numbers[0..12]: [319, 319, 326, 312, 324, 319, 326, 298, 312, 304, 325, 326]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496710435Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208342024-02-08T06:47:42.304Z{"msg":"[2]R generation[0..12]: [187, 187, 190, 183, 188, 187, 190, 176, 183, 179, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496742942Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208352024-02-08T06:47:42.305Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496760291Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208362024-02-08T06:47:42.305Z{"msg":"Max found gen is 191","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496786509Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208372024-02-08T06:47:42.306Z{"msg":"Generation requested: 192 >= found:191","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496801198Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208382024-02-08T06:47:42.306Z{"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496815207Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208392024-02-08T06:47:42.306Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496828666Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"":"downstairs","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208402024-02-08T06:47:42.306Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496857374Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208412024-02-08T06:47:42.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496891721Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"0","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208422024-02-08T06:47:42.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496921649Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"1","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208432024-02-08T06:47:42.307Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496936058Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"client":"2","":"downstairs","session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208442024-02-08T06:47:42.307Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.496968546Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208452024-02-08T06:47:42.307Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.497016352Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208462024-02-08T06:47:42.307Z{"msg":"280a6cd3-3bb4-47db-9ed7-29423d1d83c6 is now active with session: d276a78c-5302-4a1f-a0b9-1ca0f9d81f13","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.49704114Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208472024-02-08T06:47:42.308Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.497054129Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906,"session_id":"d276a78c-5302-4a1f-a0b9-1ca0f9d81f13"}
208482024-02-08T06:47:42.308Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:41.497078827Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906}
208492024-02-08T06:47:42.308ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208502024-02-08T06:47:42.308ZLoading write count information from file "/tmp/test_repair_verify.data"
208512024-02-08T06:47:42.308ZRun Repair workload
208522024-02-08T06:47:42.309Z01/30 Write block 72 len 28672 data: 12 12 14 13 14 13 13
208532024-02-08T06:47:42.309Z02/30 Read block 287 len 32768
208542024-02-08T06:47:42.309Z03/30 Read block 347 len 8192
208552024-02-08T06:47:42.310Z04/30 Write block 564 len 20480 data: 14 14 14 15 16
208562024-02-08T06:47:42.310Z05/30 Write block 369 len 4096 data: 16
208572024-02-08T06:47:42.310Z06/30 Flush
208582024-02-08T06:47:42.311Z07/30 Write block 23 len 24576 data: 14 18 19 19 19 18
208592024-02-08T06:47:42.311Z08/30 Read block 324 len 40960
208602024-02-08T06:47:42.322Z09/30 Write block 323 len 36864 data: 16 15 18 16 12 9 8 10 14
208612024-02-08T06:47:42.322Z10/30 Flush
208622024-02-08T06:47:42.322Z11/30 Flush
208632024-02-08T06:47:42.322Z12/30 Write block 80 len 36864 data: 16 16 18 20 19 18 16 16 15
208642024-02-08T06:47:42.322Z13/30 Read block 142 len 28672
208652024-02-08T06:47:42.322Z14/30 Write block 550 len 8192 data: 29 30
208662024-02-08T06:47:42.322Z15/30 Write block 335 len 4096 data: 22
208672024-02-08T06:47:42.323Z16/30 Write block 430 len 16384 data: 14 13 14 11
208682024-02-08T06:47:42.323Z17/30 Write block 239 len 12288 data: 22 19 22
208692024-02-08T06:47:42.323Z18/30 Write block 120 len 16384 data: 12 11 10 11
208702024-02-08T06:47:42.323Z19/30 Read block 76 len 12288
208712024-02-08T06:47:42.323Z20/30 Flush
208722024-02-08T06:47:42.323Z21/30 Write block 330 len 20480 data: 11 15 16 19 22
208732024-02-08T06:47:42.323Z22/30 Read block 265 len 16384
208742024-02-08T06:47:42.323Z23/30 Read block 360 len 28672
208752024-02-08T06:47:42.323Z24/30 Read block 587 len 4096
208762024-02-08T06:47:42.323Z25/30 Read block 193 len 8192
208772024-02-08T06:47:42.323Z26/30 Write block 99 len 36864 data: 21 22 17 15 15 15 12 11 10
208782024-02-08T06:47:42.324Z27/30 Write block 479 len 8192 data: 13 10
208792024-02-08T06:47:42.324Z28/30 Read block 100 len 4096
208802024-02-08T06:47:42.324Z29/30 Write block 12 len 12288 data: 14 15 14
208812024-02-08T06:47:42.324Z30/30 Write block 256 len 24576 data: 13 12 14 17 17 22
208822024-02-08T06:47:42.324Z----------------------------------------------------------------
208832024-02-08T06:47:42.324Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
208842024-02-08T06:47:42.324ZStates: Active Active Active
208852024-02-08T06:47:42.324ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208862024-02-08T06:47:42.324Z 1 Acked 1000 Write 7 Done Done Done false
208872024-02-08T06:47:42.324Z 2 Acked 1001 Read 8 Done Done Done false
208882024-02-08T06:47:42.324Z 3 Acked 1002 Read 2 Sent Done Done false
208892024-02-08T06:47:42.324Z 4 Acked 1003 Write 5 Sent Done Done false
208902024-02-08T06:47:42.324Z 5 Acked 1004 Write 1 Sent Done Done false
208912024-02-08T06:47:42.324Z 6 Acked 1005 Flush 0 Sent Done Done false
208922024-02-08T06:47:42.325Z 7 Acked 1006 Write 6 Sent Done Done false
208932024-02-08T06:47:42.325Z 8 Acked 1007 Read 10 Sent Done Done false
208942024-02-08T06:47:42.325Z 9 Acked 1008 Write 9 Sent Done Done false
208952024-02-08T06:47:42.325Z 10 Acked 1009 Flush 0 Sent Done Done false
208962024-02-08T06:47:42.325Z 11 Acked 1010 Flush 0 Sent Done Done false
208972024-02-08T06:47:42.325Z 12 Acked 1011 Write 9 Sent Done Done false
208982024-02-08T06:47:42.325Z 13 Acked 1012 Read 7 Sent Done Done false
208992024-02-08T06:47:42.325Z 14 Acked 1013 Write 2 Sent Done Done false
209002024-02-08T06:47:42.325Z 15 Acked 1014 Write 1 Sent Done Done false
209012024-02-08T06:47:42.325Z 16 Acked 1015 Write 4 Sent Done Done false
209022024-02-08T06:47:42.325Z 17 Acked 1016 Write 3 Sent Done Done false
209032024-02-08T06:47:42.325Z 18 Acked 1017 Write 4 Sent Done Done false
209042024-02-08T06:47:42.325Z 19 Acked 1018 Read 3 Sent Done Done false
209052024-02-08T06:47:42.326Z 20 Acked 1019 Flush 0 Sent Done Done false
209062024-02-08T06:47:42.326Z 21 Acked 1020 Write 5 Sent Done Done false
209072024-02-08T06:47:42.326Z 22 Acked 1021 Read 4 Sent Done Done false
209082024-02-08T06:47:42.326Z 23 Acked 1022 Read 7 Sent Done Done false
209092024-02-08T06:47:42.326Z 24 Acked 1023 Read 1 Sent Done Done false
209102024-02-08T06:47:42.326Z 25 Acked 1024 Read 2 Sent Done Done false
209112024-02-08T06:47:42.326Z 26 Acked 1025 Write 9 Sent Done Done false
209122024-02-08T06:47:42.326Z 27 Acked 1026 Write 2 Sent Done Done false
209132024-02-08T06:47:42.326Z 28 Acked 1027 Read 1 Sent Done Done false
209142024-02-08T06:47:42.326Z 29 Acked 1028 Write 3 Sent Done Done false
209152024-02-08T06:47:42.326Z 30 Acked 1029 Write 6 Sent Sent Sent false
209162024-02-08T06:47:42.326Z STATES DS:0 DS:1 DS:2 TOTAL
209172024-02-08T06:47:42.326Z New 0 0 0 0
209182024-02-08T06:47:42.327Z Sent 28 1 1 30
209192024-02-08T06:47:42.327Z Done 2 29 29 60
209202024-02-08T06:47:42.327Z Skipped 0 0 0 0
209212024-02-08T06:47:42.327Z Error 0 0 0 0
209222024-02-08T06:47:42.327ZLast Flush: 0 1019 1019
209232024-02-08T06:47:42.327ZDownstairs last five completed:
209242024-02-08T06:47:42.327ZUpstairs last five completed: 30 29 28 27 26
209252024-02-08T06:47:42.327Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:41.526683425Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1906}
209262024-02-08T06:47:42.327ZWrote out file "/tmp/test_repair_verify.data"
209272024-02-08T06:47:42.330Z
209282024-02-08T06:47:42.330ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209292024-02-08T06:47:42.344ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209302024-02-08T06:47:42.344Z 0 000-019 187 187 187 319 319 319 F T T <---
209312024-02-08T06:47:42.344Z 1 020-039 187 192 192 319 328 328 F F F <---
209322024-02-08T06:47:42.344Z 2 040-059 190 190 190 326 326 326 F F F
209332024-02-08T06:47:42.344Z 3 060-079 183 192 192 312 327 327 T F F <---
209342024-02-08T06:47:42.345Z 4 080-099 188 192 192 324 330 330 F T T <---
209352024-02-08T06:47:42.345Z 5 100-119 187 187 187 319 319 319 F T T <---
209362024-02-08T06:47:42.345Z 6 120-139 190 192 192 326 330 330 F F F <---
209372024-02-08T06:47:42.345Z 7 140-159 176 176 176 298 298 298 F F F
209382024-02-08T06:47:42.348Z 8 160-179 183 183 183 312 312 312 F F F
209392024-02-08T06:47:42.348Z 9 180-199 179 179 179 304 304 304 F F F
209402024-02-08T06:47:42.348Z 10 200-219 189 189 189 325 325 325 F F F
209412024-02-08T06:47:42.348Z 11 220-239 190 192 192 326 330 330 F F F <---
209422024-02-08T06:47:42.348Z 12 240-259 190 192 192 326 330 330 F T T <---
209432024-02-08T06:47:42.348Z 13 260-279 190 190 190 326 326 326 F T T <---
209442024-02-08T06:47:42.349Z 14 280-299 190 190 190 326 326 326 F F F
209452024-02-08T06:47:42.349Z 15 300-319 187 187 187 319 319 319 F F F
209462024-02-08T06:47:42.349Z 16 320-339 190 192 192 326 330 330 F T T <---
209472024-02-08T06:47:42.349Z 17 340-359 190 190 190 326 326 326 F F F
209482024-02-08T06:47:42.349Z 18 360-379 186 192 192 318 327 327 F F F <---
209492024-02-08T06:47:42.349Z 19 380-399 185 185 185 317 317 317 F F F
209502024-02-08T06:47:42.349Z 20 400-419 190 190 190 326 326 326 F F F
209512024-02-08T06:47:42.349Z 21 420-439 190 192 192 326 330 330 F F F <---
209522024-02-08T06:47:42.349Z 22 440-459 188 188 188 324 324 324 F F F
209532024-02-08T06:47:42.349Z 23 460-479 190 190 190 326 326 326 F T T <---
209542024-02-08T06:47:42.350Z 24 480-499 190 190 190 326 326 326 F T T <---
209552024-02-08T06:47:42.350Z 25 500-519 188 188 188 320 320 320 F F F
209562024-02-08T06:47:42.350Z 26 520-539 190 190 190 326 326 326 F F F
209572024-02-08T06:47:42.350Z 27 540-559 190 192 192 326 330 330 F F F <---
209582024-02-08T06:47:42.350Z 28 560-579 184 192 192 315 327 327 F F F <---
209592024-02-08T06:47:42.350Z 29 580-599 184 184 184 315 315 315 F F F
209602024-02-08T06:47:42.350ZMax gen: 192, Max flush: 330
209612024-02-08T06:47:42.350ZError: Difference in extent metadata found!
209622024-02-08T06:47:42.350ZA Difference in extent metadata is expected here
209632024-02-08T06:47:42.350ZOn loop 96
209642024-02-08T06:47:42.350Z
209652024-02-08T06:47:42.353ZVerifying data now
209662024-02-08T06:47:48.116ZNo change after verify
209672024-02-08T06:47:48.117ZLoop: 96 Downstairs dump after verify (and repair):
209682024-02-08T06:47:48.130ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209692024-02-08T06:47:48.130Z 0 000-019 193 193 193 331 331 331 F F F
209702024-02-08T06:47:48.130Z 1 020-039 192 192 192 328 328 328 F F F
209712024-02-08T06:47:48.130Z 2 040-059 190 190 190 326 326 326 F F F
209722024-02-08T06:47:48.130Z 3 060-079 192 192 192 327 327 327 F F F
209732024-02-08T06:47:48.137Z 4 080-099 193 193 193 331 331 331 F F F
209742024-02-08T06:47:48.137Z 5 100-119 193 193 193 331 331 331 F F F
209752024-02-08T06:47:48.137Z 6 120-139 192 192 192 330 330 330 F F F
209762024-02-08T06:47:48.137Z 7 140-159 176 176 176 298 298 298 F F F
209772024-02-08T06:47:48.137Z 8 160-179 183 183 183 312 312 312 F F F
209782024-02-08T06:47:48.137Z 9 180-199 179 179 179 304 304 304 F F F
209792024-02-08T06:47:48.137Z 10 200-219 189 189 189 325 325 325 F F F
209802024-02-08T06:47:48.137Z 11 220-239 192 192 192 330 330 330 F F F
209812024-02-08T06:47:48.137Z 12 240-259 193 193 193 331 331 331 F F F
209822024-02-08T06:47:48.138Z 13 260-279 193 193 193 331 331 331 F F F
209832024-02-08T06:47:48.138Z 14 280-299 190 190 190 326 326 326 F F F
209842024-02-08T06:47:48.138Z 15 300-319 187 187 187 319 319 319 F F F
209852024-02-08T06:47:48.138Z 16 320-339 193 193 193 331 331 331 F F F
209862024-02-08T06:47:48.138Z 17 340-359 190 190 190 326 326 326 F F F
209872024-02-08T06:47:48.138Z 18 360-379 192 192 192 327 327 327 F F F
209882024-02-08T06:47:48.138Z 19 380-399 185 185 185 317 317 317 F F F
209892024-02-08T06:47:48.138Z 20 400-419 190 190 190 326 326 326 F F F
209902024-02-08T06:47:48.138Z 21 420-439 192 192 192 330 330 330 F F F
209912024-02-08T06:47:48.138Z 22 440-459 188 188 188 324 324 324 F F F
209922024-02-08T06:47:48.138Z 23 460-479 193 193 193 331 331 331 F F F
209932024-02-08T06:47:48.138Z 24 480-499 193 193 193 331 331 331 F F F
209942024-02-08T06:47:48.138Z 25 500-519 188 188 188 320 320 320 F F F
209952024-02-08T06:47:48.138Z 26 520-539 190 190 190 326 326 326 F F F
209962024-02-08T06:47:48.138Z 27 540-559 192 192 192 330 330 330 F F F
209972024-02-08T06:47:48.138Z 28 560-579 192 192 192 327 327 327 F F F
209982024-02-08T06:47:48.139Z 29 580-599 184 184 184 315 315 315 F F F
209992024-02-08T06:47:48.139ZMax gen: 193, Max flush: 331
210002024-02-08T06:47:48.139Z
210012024-02-08T06:47:48.139ZBegin loop 97
210022024-02-08T06:47:48.139ZDownstairs to restart: 1
210032024-02-08T06:47:48.152Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.358579248Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914}
210042024-02-08T06:47:48.152Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
210052024-02-08T06:47:48.152Z,"time":"2024-02-08T06:47:47.358919493Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914}
210062024-02-08T06:47:48.156Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.3589531Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914}
210072024-02-08T06:47:48.157Z{"msg":"Crucible eebabb55-845a-485e-8d26-0d88f2cd5828 has session id: a5544152-140a-4b1d-adef-d6f9d29bffd3","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.358982548Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210082024-02-08T06:47:48.157Z{"msg":"Upstairs opts: Upstairs UUID: eebabb55-845a-485e-8d26-0d88f2cd5828, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359016366Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210092024-02-08T06:47:48.157Z{"msg":"Crucible stats registered with UUID: eebabb55-845a-485e-8d26-0d88f2cd5828","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359043424Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210102024-02-08T06:47:48.157Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359059912Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914}
210112024-02-08T06:47:48.157Z{"msg":"eebabb55-845a-485e-8d26-0d88f2cd5828 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359197342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210122024-02-08T06:47:48.157Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359230939Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210132024-02-08T06:47:48.157Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359263217Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210142024-02-08T06:47:48.157Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359305764Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210152024-02-08T06:47:48.157Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359367389Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210162024-02-08T06:47:48.157Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359404297Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210172024-02-08T06:47:48.157Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359435014Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210182024-02-08T06:47:48.157Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359476071Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"io task","client":"0","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210192024-02-08T06:47:48.157Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359503989Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"io task","client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210202024-02-08T06:47:48.157Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359539536Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"io task","client":"2","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210212024-02-08T06:47:48.157Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359564925Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"io task","client":"2","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210222024-02-08T06:47:48.158Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359608231Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"io task","client":"0","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210232024-02-08T06:47:48.158Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:47.359658937Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"io task","client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210242024-02-08T06:47:48.158Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:47.359704084Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"io task","client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210252024-02-08T06:47:48.158Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:47.359735822Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210262024-02-08T06:47:48.158Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.3597594Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210272024-02-08T06:47:48.158Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359796477Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210282024-02-08T06:47:48.158Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.359884321Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"io task","client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210292024-02-08T06:47:48.158Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.360303619Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210302024-02-08T06:47:48.158Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.360331417Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210312024-02-08T06:47:48.158Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.360382923Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210322024-02-08T06:47:48.158Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.360411481Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210332024-02-08T06:47:48.158Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.360884755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210342024-02-08T06:47:48.158Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.361032594Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210352024-02-08T06:47:48.158Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.361434904Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210362024-02-08T06:47:48.158Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.36148365Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210372024-02-08T06:47:48.158Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.361562984Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210382024-02-08T06:47:48.159Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.361606451Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210392024-02-08T06:47:48.159Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.361640029Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210402024-02-08T06:47:48.159Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:47.361675146Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210412024-02-08T06:47:49.154Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.361144398Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"io task","client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210422024-02-08T06:47:49.155Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.361347442Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"io task","client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210432024-02-08T06:47:49.160Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.362136293Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210442024-02-08T06:47:49.160Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.362208418Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210452024-02-08T06:47:49.160Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.362675223Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210462024-02-08T06:47:49.160Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.36323904Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210472024-02-08T06:47:49.160Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363279017Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210482024-02-08T06:47:49.160Z{"msg":"[0]R flush_numbers[0..12]: [331, 328, 326, 327, 331, 331, 330, 298, 312, 304, 325, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363295526Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210492024-02-08T06:47:49.160Z{"msg":"[0]R generation[0..12]: [193, 192, 190, 192, 193, 193, 192, 176, 183, 179, 189, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363317974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210502024-02-08T06:47:49.160Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363333133Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210512024-02-08T06:47:49.160Z{"msg":"[1]R flush_numbers[0..12]: [331, 328, 326, 327, 331, 331, 330, 298, 312, 304, 325, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363353942Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210522024-02-08T06:47:49.160Z{"msg":"[1]R generation[0..12]: [193, 192, 190, 192, 193, 193, 192, 176, 183, 179, 189, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363368811Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210532024-02-08T06:47:49.161Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363389929Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210542024-02-08T06:47:49.161Z{"msg":"[2]R flush_numbers[0..12]: [331, 328, 326, 327, 331, 331, 330, 298, 312, 304, 325, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363404728Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210552024-02-08T06:47:49.161Z{"msg":"[2]R generation[0..12]: [193, 192, 190, 192, 193, 193, 192, 176, 183, 179, 189, 192]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
210562024-02-08T06:47:49.161Z,"time":"2024-02-08T06:47:48.363425086Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210572024-02-08T06:47:49.161Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363446765Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210582024-02-08T06:47:49.161Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363472173Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210592024-02-08T06:47:49.161Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363489702Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210602024-02-08T06:47:49.161Z{"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.36351383Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210612024-02-08T06:47:49.161Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363531018Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"":"downstairs","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210622024-02-08T06:47:49.161Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363556027Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210632024-02-08T06:47:49.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363571635Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"0","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210642024-02-08T06:47:49.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363596774Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"1","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210652024-02-08T06:47:49.161Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363615122Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"client":"2","":"downstairs","session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210662024-02-08T06:47:49.161Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.36364016Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210672024-02-08T06:47:49.161Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363656079Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210682024-02-08T06:47:49.161Z{"msg":"eebabb55-845a-485e-8d26-0d88f2cd5828 is now active with session: a5544152-140a-4b1d-adef-d6f9d29bffd3","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363680617Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210692024-02-08T06:47:49.162Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363695786Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914,"session_id":"a5544152-140a-4b1d-adef-d6f9d29bffd3"}
210702024-02-08T06:47:49.162Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:48.363719204Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914}
210712024-02-08T06:47:49.162ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
210722024-02-08T06:47:49.162ZLoading write count information from file "/tmp/test_repair_verify.data"
210732024-02-08T06:47:49.162ZRun Repair workload
210742024-02-08T06:47:49.162Z01/30 Write block 485 len 4096 data: 17
210752024-02-08T06:47:49.165Z02/30 Write block 260 len 28672 data: 18 23 22 18 18 18 18
210762024-02-08T06:47:49.165Z03/30 Read block 128 len 36864
210772024-02-08T06:47:49.166Z04/30 Write block 238 len 4096 data: 21
210782024-02-08T06:47:49.166Z05/30 Read block 524 len 32768
210792024-02-08T06:47:49.166Z06/30 Read block 292 len 20480
210802024-02-08T06:47:49.166Z07/30 Write block 551 len 28672 data: 31 29 27 24 23 24 16
210812024-02-08T06:47:49.166Z08/30 Write block 144 len 32768 data: 9 7 6 9 9 12 10 13
210822024-02-08T06:47:49.166Z09/30 Read block 94 len 32768
210832024-02-08T06:47:49.169Z10/30 Write block 315 len 24576 data: 10 11 10 13 15 20
210842024-02-08T06:47:49.169Z11/30 Write block 135 len 36864 data: 11 12 16 15 13 10 10 9 10
210852024-02-08T06:47:49.169Z12/30 Read block 234 len 12288
210862024-02-08T06:47:49.169Z13/30 Read block 102 len 4096
210872024-02-08T06:47:49.172Z14/30 Read block 174 len 40960
210882024-02-08T06:47:49.172Z15/30 Read block 274 len 16384
210892024-02-08T06:47:49.172Z16/30 Write block 358 len 8192 data: 15 16
210902024-02-08T06:47:49.172Z17/30 Read block 213 len 20480
210912024-02-08T06:47:49.175Z18/30 Write block 132 len 20480 data: 12 12 11 12 13
210922024-02-08T06:47:49.175Z19/30 Read block 32 len 32768
210932024-02-08T06:47:49.175Z20/30 Write block 134 len 36864 data: 12 13 14 17 16 14 11 11 10
210942024-02-08T06:47:49.178Z21/30 Read block 187 len 4096
210952024-02-08T06:47:49.180Z22/30 Read block 249 len 28672
210962024-02-08T06:47:49.181Z23/30 Write block 204 len 4096 data: 15
210972024-02-08T06:47:49.181Z24/30 Read block 494 len 8192
210982024-02-08T06:47:49.181Z25/30 Write block 123 len 32768 data: 12 10 13 11 9 11 11 12
210992024-02-08T06:47:49.181Z26/30 Write block 44 len 4096 data: 8
211002024-02-08T06:47:49.184Z27/30 Read block 325 len 8192
211012024-02-08T06:47:49.184Z28/30 Read block 129 len 32768
211022024-02-08T06:47:49.184Z29/30 Write block 529 len 40960 data: 15 15 15 19 17 17 18 18 17 17
211032024-02-08T06:47:49.187Z30/30 Write block 355 len 20480 data: 15 12 14 16 17
211042024-02-08T06:47:49.187Z----------------------------------------------------------------
211052024-02-08T06:47:49.187Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
211062024-02-08T06:47:49.187ZStates: Active Active Active
211072024-02-08T06:47:49.187ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211082024-02-08T06:47:49.187Z 1 Acked 1000 Write 1 Done Sent Done false
211092024-02-08T06:47:49.187Z 2 Acked 1001 Write 7 Done Sent Done false
211102024-02-08T06:47:49.187Z 3 Acked 1002 Read 9 Done Sent Done false
211112024-02-08T06:47:49.187Z 4 Acked 1003 Write 1 Done Sent Done false
211122024-02-08T06:47:49.188Z 5 Acked 1004 Read 8 Done Sent Done false
211132024-02-08T06:47:49.188Z 6 Acked 1005 Read 5 Done Sent Done false
211142024-02-08T06:47:49.188Z 7 Acked 1006 Write 7 Done Sent Done false
211152024-02-08T06:47:49.188Z 8 Acked 1007 Write 8 Done Sent Done false
211162024-02-08T06:47:49.188Z 9 Acked 1008 Read 8 Done Sent Done false
211172024-02-08T06:47:49.188Z 10 Acked 1009 Write 6 Done Sent Done false
211182024-02-08T06:47:49.188Z 11 Acked 1010 Write 9 Done Sent Done false
211192024-02-08T06:47:49.188Z 12 Acked 1011 Read 3 Done Sent Done false
211202024-02-08T06:47:49.188Z 13 Acked 1012 Read 1 Done Sent Done false
211212024-02-08T06:47:49.188Z 14 Acked 1013 Read 10 Done Sent Done false
211222024-02-08T06:47:49.188Z 15 Acked 1014 Read 4 Done Sent Done false
211232024-02-08T06:47:49.188Z 16 Acked 1015 Write 2 Done Sent Done false
211242024-02-08T06:47:49.188Z 17 Acked 1016 Read 5 Done Sent Done false
211252024-02-08T06:47:49.188Z 18 Acked 1017 Write 5 Done Sent Done false
211262024-02-08T06:47:49.188Z 19 Acked 1018 Read 8 Done Sent Done false
211272024-02-08T06:47:49.188Z 20 Acked 1019 Write 9 Done Sent Done false
211282024-02-08T06:47:49.189Z 21 Acked 1020 Read 1 Done Sent Done false
211292024-02-08T06:47:49.189Z 22 Acked 1021 Read 7 Done Sent Done false
211302024-02-08T06:47:49.189Z 23 Acked 1022 Write 1 Done Sent Done false
211312024-02-08T06:47:49.189Z 24 Acked 1023 Read 2 Done Sent Done false
211322024-02-08T06:47:49.189Z 25 Acked 1024 Write 8 Done Sent Done false
211332024-02-08T06:47:49.189Z 26 Acked 1025 Write 1 Done Sent Done false
211342024-02-08T06:47:49.189Z 27 Acked 1026 Read 2 Done Sent Done false
211352024-02-08T06:47:49.189Z 28 Acked 1027 Read 8 Done Sent Done false
211362024-02-08T06:47:49.189Z 29 Acked 1028 Write 10 Sent Sent Sent false
211372024-02-08T06:47:49.189Z 30 Acked 1029 Write 5 Sent Sent Sent false
211382024-02-08T06:47:49.189Z STATES DS:0 DS:1 DS:2 TOTAL
211392024-02-08T06:47:49.189Z New 0 0 0 0
211402024-02-08T06:47:49.189Z Sent 2 30 2 34
211412024-02-08T06:47:49.189Z Done 28 0 28 56
211422024-02-08T06:47:49.189Z Skipped 0 0 0 0
211432024-02-08T06:47:49.190Z Error 0 0 0 0
211442024-02-08T06:47:49.190ZLast Flush: 0 0 0
211452024-02-08T06:47:49.190ZDownstairs last five completed:
211462024-02-08T06:47:49.190ZUpstairs last five completed: 30 29 28 27 26
211472024-02-08T06:47:49.190Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:48.391786277Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1914}
211482024-02-08T06:47:49.190ZWrote out file "/tmp/test_repair_verify.data"
211492024-02-08T06:47:49.196Z
211502024-02-08T06:47:49.196ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211512024-02-08T06:47:49.216ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211522024-02-08T06:47:49.216Z 0 000-019 193 193 193 331 331 331 F F F
211532024-02-08T06:47:49.216Z 1 020-039 192 192 192 328 328 328 F F F
211542024-02-08T06:47:49.216Z 2 040-059 190 190 190 326 326 326 T F T <---
211552024-02-08T06:47:49.216Z 3 060-079 192 192 192 327 327 327 F F F
211562024-02-08T06:47:49.216Z 4 080-099 193 193 193 331 331 331 F F F
211572024-02-08T06:47:49.216Z 5 100-119 193 193 193 331 331 331 F F F
211582024-02-08T06:47:49.216Z 6 120-139 192 192 192 330 330 330 T F T <---
211592024-02-08T06:47:49.216Z 7 140-159 176 176 176 298 298 298 T F T <---
211602024-02-08T06:47:49.216Z 8 160-179 183 183 183 312 312 312 F F F
211612024-02-08T06:47:49.216Z 9 180-199 179 179 179 304 304 304 F F F
211622024-02-08T06:47:49.217Z 10 200-219 189 189 189 325 325 325 T F T <---
211632024-02-08T06:47:49.217Z 11 220-239 192 192 192 330 330 330 T F T <---
211642024-02-08T06:47:49.220Z 12 240-259 193 193 193 331 331 331 F F F
211652024-02-08T06:47:49.220Z 13 260-279 193 193 193 331 331 331 T F T <---
211662024-02-08T06:47:49.220Z 14 280-299 190 190 190 326 326 326 F F F
211672024-02-08T06:47:49.220Z 15 300-319 187 187 187 319 319 319 T F T <---
211682024-02-08T06:47:49.220Z 16 320-339 193 193 193 331 331 331 T F T <---
211692024-02-08T06:47:49.220Z 17 340-359 190 190 190 326 326 326 T F T <---
211702024-02-08T06:47:49.220Z 18 360-379 192 192 192 327 327 327 F F F
211712024-02-08T06:47:49.220Z 19 380-399 185 185 185 317 317 317 F F F
211722024-02-08T06:47:49.220Z 20 400-419 190 190 190 326 326 326 F F F
211732024-02-08T06:47:49.220Z 21 420-439 192 192 192 330 330 330 F F F
211742024-02-08T06:47:49.220Z 22 440-459 188 188 188 324 324 324 F F F
211752024-02-08T06:47:49.220Z 23 460-479 193 193 193 331 331 331 F F F
211762024-02-08T06:47:49.220Z 24 480-499 193 193 193 331 331 331 T F T <---
211772024-02-08T06:47:49.220Z 25 500-519 188 188 188 320 320 320 F F F
211782024-02-08T06:47:49.220Z 26 520-539 190 190 190 326 326 326 T F T <---
211792024-02-08T06:47:49.221Z 27 540-559 192 192 192 330 330 330 T F T <---
211802024-02-08T06:47:49.221Z 28 560-579 192 192 192 327 327 327 F F F
211812024-02-08T06:47:49.221Z 29 580-599 184 184 184 315 315 315 F F F
211822024-02-08T06:47:49.221ZMax gen: 193, Max flush: 331
211832024-02-08T06:47:49.221ZError: Difference in extent metadata found!
211842024-02-08T06:47:49.221ZA Difference in extent metadata is expected here
211852024-02-08T06:47:49.221ZOn loop 97
211862024-02-08T06:47:49.221Z
211872024-02-08T06:47:49.221ZVerifying data now
211882024-02-08T06:47:53.840ZNo change after verify
211892024-02-08T06:47:53.840ZLoop: 97 Downstairs dump after verify (and repair):
211902024-02-08T06:47:53.855ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211912024-02-08T06:47:53.855Z 0 000-019 193 193 193 331 331 331 F F F
211922024-02-08T06:47:53.855Z 1 020-039 192 192 192 328 328 328 F F F
211932024-02-08T06:47:53.855Z 2 040-059 194 194 194 332 332 332 F F F
211942024-02-08T06:47:53.855Z 3 060-079 192 192 192 327 327 327 F F F
211952024-02-08T06:47:53.855Z 4 080-099 193 193 193 331 331 331 F F F
211962024-02-08T06:47:53.855Z 5 100-119 193 193 193 331 331 331 F F F
211972024-02-08T06:47:53.855Z 6 120-139 194 194 194 332 332 332 F F F
211982024-02-08T06:47:53.855Z 7 140-159 194 194 194 332 332 332 F F F
211992024-02-08T06:47:53.855Z 8 160-179 183 183 183 312 312 312 F F F
212002024-02-08T06:47:53.856Z 9 180-199 179 179 179 304 304 304 F F F
212012024-02-08T06:47:53.856Z 10 200-219 194 194 194 332 332 332 F F F
212022024-02-08T06:47:53.859Z 11 220-239 194 194 194 332 332 332 F F F
212032024-02-08T06:47:53.859Z 12 240-259 193 193 193 331 331 331 F F F
212042024-02-08T06:47:53.859Z 13 260-279 194 194 194 332 332 332 F F F
212052024-02-08T06:47:53.859Z 14 280-299 190 190 190 326 326 326 F F F
212062024-02-08T06:47:53.859Z 15 300-319 194 194 194 332 332 332 F F F
212072024-02-08T06:47:53.859Z 16 320-339 194 194 194 332 332 332 F F F
212082024-02-08T06:47:53.859Z 17 340-359 194 194 194 332 332 332 F F F
212092024-02-08T06:47:53.859Z 18 360-379 192 192 192 327 327 327 F F F
212102024-02-08T06:47:53.860Z 19 380-399 185 185 185 317 317 317 F F F
212112024-02-08T06:47:53.860Z 20 400-419 190 190 190 326 326 326 F F F
212122024-02-08T06:47:53.860Z 21 420-439 192 192 192 330 330 330 F F F
212132024-02-08T06:47:53.860Z 22 440-459 188 188 188 324 324 324 F F F
212142024-02-08T06:47:53.860Z 23 460-479 193 193 193 331 331 331 F F F
212152024-02-08T06:47:53.860Z 24 480-499 194 194 194 332 332 332 F F F
212162024-02-08T06:47:53.860Z 25 500-519 188 188 188 320 320 320 F F F
212172024-02-08T06:47:53.860Z 26 520-539 194 194 194 332 332 332 F F F
212182024-02-08T06:47:53.860Z 27 540-559 194 194 194 332 332 332 F F F
212192024-02-08T06:47:53.860Z 28 560-579 192 192 192 327 327 327 F F F
212202024-02-08T06:47:53.860Z 29 580-599 184 184 184 315 315 315 F F F
212212024-02-08T06:47:53.860ZMax gen: 194, Max flush: 332
212222024-02-08T06:47:53.860Z
212232024-02-08T06:47:53.860ZBegin loop 98
212242024-02-08T06:47:53.860ZDownstairs to restart: 0
212252024-02-08T06:47:53.875Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.081779874Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922}
212262024-02-08T06:47:53.875Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082046974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922}
212272024-02-08T06:47:53.880Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082080981Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922}
212282024-02-08T06:47:53.880Z{"msg":"Crucible runtime is spawned
212292024-02-08T06:47:53.881ZCrucible 3dd0dfe1-7fb0-4fd6-ae0a-9082bdf38baa has session id: 38ad88a3-c71a-4b64-be06-716daccc3d58","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082112869Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212302024-02-08T06:47:53.881Z{"msg":"Upstairs opts: Upstairs UUID: 3dd0dfe1-7fb0-4fd6-ae0a-9082bdf38baa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082145186Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212312024-02-08T06:47:53.881Z{"msg":"Crucible stats registered with UUID: 3dd0dfe1-7fb0-4fd6-ae0a-9082bdf38baa","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082170484Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212322024-02-08T06:47:53.881Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082193573Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922}
212332024-02-08T06:47:53.881Z{"msg":"3dd0dfe1-7fb0-4fd6-ae0a-9082bdf38baa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082322593Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212342024-02-08T06:47:53.881Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082351381Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212352024-02-08T06:47:53.881Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082374919Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212362024-02-08T06:47:53.881Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082414966Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212372024-02-08T06:47:53.881Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082441864Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212382024-02-08T06:47:53.881Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082472902Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212392024-02-08T06:47:53.881Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.08248944Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212402024-02-08T06:47:53.881Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082516608Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"io task","client":"0","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212412024-02-08T06:47:53.881Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082573224Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"io task","client":"2","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212422024-02-08T06:47:53.881Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082617451Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"io task","client":"1","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212432024-02-08T06:47:53.881Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082650898Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"io task","client":"1","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212442024-02-08T06:47:53.882Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082742162Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"io task","client":"0","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212452024-02-08T06:47:53.882Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.082799277Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"io task","client":"2","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212462024-02-08T06:47:53.882Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.083379983Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212472024-02-08T06:47:53.882Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.083436179Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212482024-02-08T06:47:53.882Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.083494675Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212492024-02-08T06:47:53.882Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.083518033Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212502024-02-08T06:47:53.882Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.083948521Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212512024-02-08T06:47:53.882Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.0840943Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212522024-02-08T06:47:53.882Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.084446134Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212532024-02-08T06:47:53.882Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.084504079Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212542024-02-08T06:47:53.882Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.084531837Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212552024-02-08T06:47:53.882Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.084555565Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212562024-02-08T06:47:53.882Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.084570224Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212572024-02-08T06:47:53.882Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.084586613Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212582024-02-08T06:47:53.882Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.084615721Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212592024-02-08T06:47:53.882Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.084636629Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212602024-02-08T06:47:53.883Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085121673Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212612024-02-08T06:47:53.887Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085608456Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212622024-02-08T06:47:53.887Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085650043Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212632024-02-08T06:47:53.887Z{"msg":"[0]R flush_numbers[0..12]: [331, 328, 332, 327, 331, 331, 332, 332, 312, 304, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085668032Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212642024-02-08T06:47:53.887Z{"msg":"[0]R generation[0..12]: [193, 192, 194, 192, 193, 193, 194, 194, 183, 179, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.08568993Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212652024-02-08T06:47:53.887Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085704929Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212662024-02-08T06:47:53.887Z{"msg":"[1]R flush_numbers[0..12]: [331, 328, 332, 327, 331, 331, 332, 332, 312, 304, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085725718Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212672024-02-08T06:47:53.887Z{"msg":"[1]R generation[0..12]: [193, 192, 194, 192, 193, 193, 194, 194, 183, 179, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085740006Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212682024-02-08T06:47:53.887Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085760535Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212692024-02-08T06:47:53.888Z{"msg":"[2]R flush_numbers[0..12]: [331, 328, 332, 327, 331, 331, 332, 332, 312, 304, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085775124Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212702024-02-08T06:47:53.888Z{"msg":"[2]R generation[0..12]: [193, 192, 194, 192, 193, 193, 194, 194, 183, 179, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085795692Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212712024-02-08T06:47:53.888ZWait for a query_work_queue command to finish before sending IO
212722024-02-08T06:47:53.888Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085814391Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212732024-02-08T06:47:53.888Z{"msg":"Max found gen is 195","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085834239Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212742024-02-08T06:47:53.888Z{"msg":"Generation requested: 196 >= found:195","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085848058Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212752024-02-08T06:47:53.888Z{"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085867367Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212762024-02-08T06:47:53.888Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085881136Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"":"downstairs","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212772024-02-08T06:47:53.888Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085912383Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212782024-02-08T06:47:53.888Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085937702Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"0","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212792024-02-08T06:47:53.888Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.085978579Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"1","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212802024-02-08T06:47:53.888Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.086004786Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"client":"2","":"downstairs","session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212812024-02-08T06:47:53.888Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.086030324Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212822024-02-08T06:47:53.888Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.086059432Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212832024-02-08T06:47:53.889Z{"msg":"3dd0dfe1-7fb0-4fd6-ae0a-9082bdf38baa is now active with session: 38ad88a3-c71a-4b64-be06-716daccc3d58","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.086073761Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212842024-02-08T06:47:53.889Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.08609527Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922,"session_id":"38ad88a3-c71a-4b64-be06-716daccc3d58"}
212852024-02-08T06:47:53.889Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:53.086119318Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922}
212862024-02-08T06:47:53.889ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
212872024-02-08T06:47:53.889ZLoading write count information from file "/tmp/test_repair_verify.data"
212882024-02-08T06:47:53.889ZRun Repair workload
212892024-02-08T06:47:53.889Z01/30 Write block 89 len 12288 data: 14 15 18
212902024-02-08T06:47:53.889Z02/30 Read block 81 len 4096
212912024-02-08T06:47:53.889Z03/30 Write block 103 len 40960 data: 16 16 13 12 11 12 13 9 11 13
212922024-02-08T06:47:53.889Z04/30 Write block 37 len 24576 data: 13 15 12 11 11 11
212932024-02-08T06:47:53.889Z05/30 Flush
212942024-02-08T06:47:53.893Z06/30 Write block 486 len 16384 data: 18 16 19 17
212952024-02-08T06:47:53.893Z07/30 Flush
212962024-02-08T06:47:53.893Z08/30 Write block 402 len 36864 data: 15 14 15 16 15 15 16 16 14
212972024-02-08T06:47:53.893Z09/30 Read block 341 len 24576
212982024-02-08T06:47:53.893Z10/30 Flush
212992024-02-08T06:47:53.893Z11/30 Write block 31 len 8192 data: 17 15
213002024-02-08T06:47:53.893Z12/30 Write block 203 len 12288 data: 15 16 17
213012024-02-08T06:47:53.896Z13/30 Read block 403 len 40960
213022024-02-08T06:47:53.896Z14/30 Read block 253 len 16384
213032024-02-08T06:47:53.896Z15/30 Read block 358 len 8192
213042024-02-08T06:47:53.897Z16/30 Read block 410 len 32768
213052024-02-08T06:47:53.900Z17/30 Flush
213062024-02-08T06:47:53.900Z18/30 Write block 13 len 36864 data: 16 15 14 14 14 15 16 18 15
213072024-02-08T06:47:53.900Z19/30 Write block 109 len 8192 data: 14 10
213082024-02-08T06:47:53.900Z20/30 Flush
213092024-02-08T06:47:53.903Z21/30 Write block 261 len 40960 data: 24 23 19 19 19 19 19 19 19 13
213102024-02-08T06:47:53.903Z22/30 Write block 97 len 16384 data: 23 21 22 23
213112024-02-08T06:47:53.906Z23/30 Write block 196 len 24576 data: 10 14 18 20 18 14
213122024-02-08T06:47:53.906Z24/30 Write block 241 len 20480 data: 23 17 19 17 17
213132024-02-08T06:47:53.906Z25/30 Write block 400 len 40960 data: 12 15 16 15 16 17 16 16 17 17
213142024-02-08T06:47:53.906Z26/30 Read block 550 len 40960
213152024-02-08T06:47:53.909Z27/30 Write block 45 len 32768 data: 6 10 7 8 7 10 11 9
213162024-02-08T06:47:53.909Z28/30 Write block 95 len 40960 data: 21 22 24 22 23 24 18 16 17 17
213172024-02-08T06:47:53.913Z29/30 Write block 362 len 36864 data: 12 12 15 16 15 18 15 17 14
213182024-02-08T06:47:53.913Z30/30 Write block 475 len 40960 data: 17 12 12 11 14 11 11 12 11 16
213192024-02-08T06:47:53.913Z----------------------------------------------------------------
213202024-02-08T06:47:53.913Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
213212024-02-08T06:47:53.913ZStates: Active Active Active
213222024-02-08T06:47:53.913ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213232024-02-08T06:47:53.913Z 1 Acked 1000 Write 3 Done Done Done false
213242024-02-08T06:47:53.913Z 2 Acked 1001 Read 1 Sent Done Done false
213252024-02-08T06:47:53.913Z 3 Acked 1002 Write 10 Sent Done Done false
213262024-02-08T06:47:53.913Z 4 Acked 1003 Write 6 Sent Done Done false
213272024-02-08T06:47:53.913Z 5 Acked 1004 Flush 0 Sent Done Done false
213282024-02-08T06:47:53.913Z 6 Acked 1005 Write 4 Sent Done Done false
213292024-02-08T06:47:53.913Z 7 Acked 1006 Flush 0 Sent Done Done false
213302024-02-08T06:47:53.913Z 8 Acked 1007 Write 9 Sent Done Done false
213312024-02-08T06:47:53.914Z 9 Acked 1008 Read 6 Sent Done Done false
213322024-02-08T06:47:53.914Z 10 Acked 1009 Flush 0 Sent Done Done false
213332024-02-08T06:47:53.914Z 11 Acked 1010 Write 2 Sent Done Done false
213342024-02-08T06:47:53.914Z 12 Acked 1011 Write 3 Sent Done Done false
213352024-02-08T06:47:53.914Z 13 Acked 1012 Read 10 Sent Done Done false
213362024-02-08T06:47:53.914Z 14 Acked 1013 Read 4 Sent Done Done false
213372024-02-08T06:47:53.914Z 15 Acked 1014 Read 2 Sent Done Done false
213382024-02-08T06:47:53.914Z 16 Acked 1015 Read 8 Sent Done Done false
213392024-02-08T06:47:53.914Z 17 Acked 1016 Flush 0 Sent Done Done false
213402024-02-08T06:47:53.914Z 18 Acked 1017 Write 9 Sent Done Done false
213412024-02-08T06:47:53.914Z 19 Acked 1018 Write 2 Sent Done Done false
213422024-02-08T06:47:53.914Z 20 Acked 1019 Flush 0 Sent Done Done false
213432024-02-08T06:47:53.914Z 21 Acked 1020 Write 10 Sent Done Done false
213442024-02-08T06:47:53.914Z 22 Acked 1021 Write 4 Sent Done Done false
213452024-02-08T06:47:53.914Z 23 Acked 1022 Write 6 Sent Done Done false
213462024-02-08T06:47:53.914Z 24 Acked 1023 Write 5 Sent Done Done false
213472024-02-08T06:47:53.915Z 25 Acked 1024 Write 10 Sent Done Done false
213482024-02-08T06:47:53.915Z 26 Acked 1025 Read 10 Sent Done Done false
213492024-02-08T06:47:53.915Z 27 Acked 1026 Write 8 Sent Done Done false
213502024-02-08T06:47:53.915Z 28 Acked 1027 Write 10 Sent Done Done false
213512024-02-08T06:47:53.915Z 29 Acked 1028 Write 9 Sent Done Done false
213522024-02-08T06:47:53.915Z 30 Acked 1029 Write 10 Sent Sent Sent false
213532024-02-08T06:47:53.915Z STATES DS:0 DS:1 DS:2 TOTAL
213542024-02-08T06:47:53.915Z New 0 0 0 0
213552024-02-08T06:47:53.915Z Sent 29 1 1 31
213562024-02-08T06:47:53.915Z Done 1 29 29 59
213572024-02-08T06:47:53.915Z Skipped 0 0 0 0
213582024-02-08T06:47:53.915Z Error 0 0 0 0
213592024-02-08T06:47:53.915ZLast Flush: 0 1019 1019
213602024-02-08T06:47:53.915ZDownstairs last five completed:
213612024-02-08T06:47:53.915ZUpstairs last five completed: 30 29 28 27 26
213622024-02-08T06:47:53.915Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:53.118313981Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1922}
213632024-02-08T06:47:53.916ZWrote out file "/tmp/test_repair_verify.data"
213642024-02-08T06:47:53.919Z
213652024-02-08T06:47:53.919ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
213662024-02-08T06:47:53.937ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213672024-02-08T06:47:53.937Z 0 000-019 193 196 196 331 337 337 F F F <---
213682024-02-08T06:47:53.938Z 1 020-039 192 196 196 328 337 337 F F F <---
213692024-02-08T06:47:53.938Z 2 040-059 194 196 196 332 333 333 F T T <---
213702024-02-08T06:47:53.938Z 3 060-079 192 192 192 327 327 327 F F F
213712024-02-08T06:47:53.938Z 4 080-099 193 196 196 331 333 333 T T T <---
213722024-02-08T06:47:53.938Z 5 100-119 193 196 196 331 337 337 F T T <---
213732024-02-08T06:47:53.938Z 6 120-139 194 194 194 332 332 332 F F F
213742024-02-08T06:47:53.938Z 7 140-159 194 194 194 332 332 332 F F F
213752024-02-08T06:47:53.941Z 8 160-179 183 183 183 312 312 312 F F F
213762024-02-08T06:47:53.941Z 9 180-199 179 179 179 304 304 304 F T T <---
213772024-02-08T06:47:53.941Z 10 200-219 194 196 196 332 336 336 F T T <---
213782024-02-08T06:47:53.941Z 11 220-239 194 194 194 332 332 332 F F F
213792024-02-08T06:47:53.941Z 12 240-259 193 193 193 331 331 331 F T T <---
213802024-02-08T06:47:53.941Z 13 260-279 194 194 194 332 332 332 F T T <---
213812024-02-08T06:47:53.941Z 14 280-299 190 190 190 326 326 326 F F F
213822024-02-08T06:47:53.942Z 15 300-319 194 194 194 332 332 332 F F F
213832024-02-08T06:47:53.942Z 16 320-339 194 194 194 332 332 332 F F F
213842024-02-08T06:47:53.942Z 17 340-359 194 194 194 332 332 332 F F F
213852024-02-08T06:47:53.942Z 18 360-379 192 192 192 327 327 327 F T T <---
213862024-02-08T06:47:53.942Z 19 380-399 185 185 185 317 317 317 F F F
213872024-02-08T06:47:53.942Z 20 400-419 190 196 196 326 335 335 F T T <---
213882024-02-08T06:47:53.942Z 21 420-439 192 192 192 330 330 330 F F F
213892024-02-08T06:47:53.942Z 22 440-459 188 188 188 324 324 324 F F F
213902024-02-08T06:47:53.942Z 23 460-479 193 193 193 331 331 331 F T T <---
213912024-02-08T06:47:53.942Z 24 480-499 194 196 196 332 334 334 F T T <---
213922024-02-08T06:47:53.942Z 25 500-519 188 188 188 320 320 320 F F F
213932024-02-08T06:47:53.942Z 26 520-539 194 194 194 332 332 332 F F F
213942024-02-08T06:47:53.942Z 27 540-559 194 194 194 332 332 332 F F F
213952024-02-08T06:47:53.942Z 28 560-579 192 192 192 327 327 327 F F F
213962024-02-08T06:47:53.942Z 29 580-599 184 184 184 315 315 315 F F F
213972024-02-08T06:47:53.943ZMax gen: 196, Max flush: 337
213982024-02-08T06:47:53.943ZError: Difference in extent metadata found!
213992024-02-08T06:47:53.943ZA Difference in extent metadata is expected here
214002024-02-08T06:47:53.943ZOn loop 98
214012024-02-08T06:47:53.943Z
214022024-02-08T06:47:53.946ZVerifying data now
214032024-02-08T06:47:58.595ZNo change after verify
214042024-02-08T06:47:58.595ZLoop: 98 Downstairs dump after verify (and repair):
214052024-02-08T06:47:58.609ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214062024-02-08T06:47:58.610Z 0 000-019 196 196 196 337 337 337 F F F
214072024-02-08T06:47:58.610Z 1 020-039 196 196 196 337 337 337 F F F
214082024-02-08T06:47:58.610Z 2 040-059 197 197 197 338 338 338 F F F
214092024-02-08T06:47:58.610Z 3 060-079 192 192 192 327 327 327 F F F
214102024-02-08T06:47:58.610Z 4 080-099 197 197 197 338 338 338 F F F
214112024-02-08T06:47:58.615Z 5 100-119 197 197 197 338 338 338 F F F
214122024-02-08T06:47:58.615Z 6 120-139 194 194 194 332 332 332 F F F
214132024-02-08T06:47:58.616Z 7 140-159 194 194 194 332 332 332 F F F
214142024-02-08T06:47:58.616Z 8 160-179 183 183 183 312 312 312 F F F
214152024-02-08T06:47:58.616Z 9 180-199 197 197 197 338 338 338 F F F
214162024-02-08T06:47:58.616Z 10 200-219 197 197 197 338 338 338 F F F
214172024-02-08T06:47:58.617Z 11 220-239 194 194 194 332 332 332 F F F
214182024-02-08T06:47:58.617Z 12 240-259 197 197 197 338 338 338 F F F
214192024-02-08T06:47:58.617Z 13 260-279 197 197 197 338 338 338 F F F
214202024-02-08T06:47:58.617Z 14 280-299 190 190 190 326 326 326 F F F
214212024-02-08T06:47:58.618Z 15 300-319 194 194 194 332 332 332 F F F
214222024-02-08T06:47:58.618Z 16 320-339 194 194 194 332 332 332 F F F
214232024-02-08T06:47:58.618Z 17 340-359 194 194 194 332 332 332 F F F
214242024-02-08T06:47:58.618Z 18 360-379 197 197 197 338 338 338 F F F
214252024-02-08T06:47:58.619Z 19 380-399 185 185 185 317 317 317 F F F
214262024-02-08T06:47:58.619Z 20 400-419 197 197 197 338 338 338 F F F
214272024-02-08T06:47:58.619Z 21 420-439 192 192 192 330 330 330 F F F
214282024-02-08T06:47:58.619Z 22 440-459 188 188 188 324 324 324 F F F
214292024-02-08T06:47:58.619Z 23 460-479 197 197 197 338 338 338 F F F
214302024-02-08T06:47:58.619Z 24 480-499 197 197 197 338 338 338 F F F
214312024-02-08T06:47:58.620Z 25 500-519 188 188 188 320 320 320 F F F
214322024-02-08T06:47:58.620Z 26 520-539 194 194 194 332 332 332 F F F
214332024-02-08T06:47:58.620Z 27 540-559 194 194 194 332 332 332 F F F
214342024-02-08T06:47:58.620Z 28 560-579 192 192 192 327 327 327 F F F
214352024-02-08T06:47:58.621Z 29 580-599 184 184 184 315 315 315 F F F
214362024-02-08T06:47:58.621ZMax gen: 197, Max flush: 338
214372024-02-08T06:47:58.621Z
214382024-02-08T06:47:58.621ZBegin loop 99
214392024-02-08T06:47:58.622ZDownstairs to restart: 1
214402024-02-08T06:47:58.626Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.832412696Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930}
214412024-02-08T06:47:58.626Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ebef7f4917e579ad4649dc0478073aaafe487733\",\n git_commit_timestamp: \"2024-02-08T06:19:45.000000000Z\",\n git_branch: \"alan/downstairs-clone\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"Crucible runtime is spawned
214422024-02-08T06:47:58.634Zx86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.832729923Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930}
214432024-02-08T06:47:58.634Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.832776709Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930}
214442024-02-08T06:47:58.634Z{"msg":"Crucible 112748bf-a170-4cf8-9368-d8e2419054e6 has session id: 502aa77e-6413-4d03-a3e0-7a80e646bb55","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.832817596Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214452024-02-08T06:47:58.634Z{"msg":"Upstairs opts: Upstairs UUID: 112748bf-a170-4cf8-9368-d8e2419054e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.832840255Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214462024-02-08T06:47:58.634Z{"msg":"Crucible stats registered with UUID: 112748bf-a170-4cf8-9368-d8e2419054e6","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.832874992Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214472024-02-08T06:47:58.634Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.83290103Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930}
214482024-02-08T06:47:58.634Z{"msg":"112748bf-a170-4cf8-9368-d8e2419054e6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.832940407Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214492024-02-08T06:47:58.634Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.832955746Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214502024-02-08T06:47:58.634Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.832992653Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214512024-02-08T06:47:58.634Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833011931Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214522024-02-08T06:47:58.634Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.83303746Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214532024-02-08T06:47:58.634Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833062038Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214542024-02-08T06:47:58.634Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833098415Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214552024-02-08T06:47:58.634Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833126383Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"io task","client":"1","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214562024-02-08T06:47:58.635Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833154291Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"io task","client":"2","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214572024-02-08T06:47:58.635Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833179409Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"io task","client":"0","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214582024-02-08T06:47:58.635Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833203927Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"io task","client":"1","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214592024-02-08T06:47:58.635Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833232365Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"io task","client":"2","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214602024-02-08T06:47:58.635Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833314029Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"io task","client":"0","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214612024-02-08T06:47:58.635Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833785253Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214622024-02-08T06:47:58.635Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833821061Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214632024-02-08T06:47:58.635Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.83383648Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214642024-02-08T06:47:58.635Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.833850629Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214652024-02-08T06:47:58.635Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.834374259Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214662024-02-08T06:47:58.635Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.834487901Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214672024-02-08T06:47:58.635Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.834911869Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214682024-02-08T06:47:58.635Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.834959335Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214692024-02-08T06:47:58.635Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.834979134Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214702024-02-08T06:47:58.635Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.834997282Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214712024-02-08T06:47:58.635Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.835011861Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214722024-02-08T06:47:58.636Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.835025411Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214732024-02-08T06:47:58.636Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.835357755Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214742024-02-08T06:47:58.636Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.835393993Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214752024-02-08T06:47:58.640Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.835889226Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214762024-02-08T06:47:58.640Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836322803Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214772024-02-08T06:47:58.641Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836356431Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214782024-02-08T06:47:58.641Z{"msg":"[0]R flush_numbers[0..12]: [337, 337, 338, 327, 338, 338, 332, 332, 312, 338, 338, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836371989Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214792024-02-08T06:47:58.641Z{"msg":"[0]R generation[0..12]: [196, 196, 197, 192, 197, 197, 194, 194, 183, 197, 197, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836404127Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214802024-02-08T06:47:58.641Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836438664Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214812024-02-08T06:47:58.641Z{"msg":"[1]R flush_numbers[0..12]: [337, 337, 338, 327, 338, 338, 332, 332, 312, 338, 338, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836472342Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214822024-02-08T06:47:58.641Z{"msg":"Wait for a query_work_queue command to finish before sending IO
214832024-02-08T06:47:58.641Z[1]R generation[0..12]: [196, 196, 197, 192, 197, 197, 194, 194, 183, 197, 197, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836505589Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214842024-02-08T06:47:58.641Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836544536Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214852024-02-08T06:47:58.641Z{"msg":"[2]R flush_numbers[0..12]: [337, 337, 338, 327, 338, 338, 332, 332, 312, 338, 338, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836560835Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214862024-02-08T06:47:58.641Z{"msg":"[2]R generation[0..12]: [196, 196, 197, 192, 197, 197, 194, 194, 183, 197, 197, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836593243Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214872024-02-08T06:47:58.641Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.83662648Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214882024-02-08T06:47:58.641Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836658748Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214892024-02-08T06:47:58.641Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836690585Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214902024-02-08T06:47:58.642Z{"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836719963Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214912024-02-08T06:47:58.642Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836752871Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"":"downstairs","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214922024-02-08T06:47:58.642Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836769309Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214932024-02-08T06:47:58.642Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836785978Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"0","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214942024-02-08T06:47:58.642Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836814816Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"1","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214952024-02-08T06:47:58.642Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836832425Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
214962024-02-08T06:47:58.642Zip-10-150-1-117.us-west-2.compute.internal","pid":1930,"client":"2","":"downstairs","session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214972024-02-08T06:47:58.642Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836863692Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214982024-02-08T06:47:58.642Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836877111Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
214992024-02-08T06:47:58.642Z{"msg":"112748bf-a170-4cf8-9368-d8e2419054e6 is now active with session: 502aa77e-6413-4d03-a3e0-7a80e646bb55","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836908339Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
215002024-02-08T06:47:58.642Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836938687Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930,"session_id":"502aa77e-6413-4d03-a3e0-7a80e646bb55"}
215012024-02-08T06:47:58.642Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-08T06:47:57.836967974Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930}
215022024-02-08T06:47:58.642ZLoading write count information from file "/tmp/test_repair_verify.data"
215032024-02-08T06:47:58.642ZRun Repair workload
215042024-02-08T06:47:58.642Z01/30 Write block 320 len 40960 data: 21 17 18 17 16 19 17 13 10 9
215052024-02-08T06:47:58.642Z02/30 Write block 583 len 12288 data: 12 12 13
215062024-02-08T06:47:58.643Z03/30 Flush
215072024-02-08T06:47:58.643Z04/30 Flush
215082024-02-08T06:47:58.643Z05/30 Write block 200 len 8192 data: 19 15
215092024-02-08T06:47:58.643Z06/30 Write block 223 len 40960 data: 16 19 18 18 19 20 16 13 10 10
215102024-02-08T06:47:58.643Z07/30 Read block 565 len 40960
215112024-02-08T06:47:58.646Z08/30 Read block 368 len 20480
215122024-02-08T06:47:58.646Z09/30 Read block 322 len 16384
215132024-02-08T06:47:58.646Z10/30 Read block 259 len 40960
215142024-02-08T06:47:58.646Z11/30 Write block 536 len 32768 data: 19 18 18 18 18 17 14 16
215152024-02-08T06:47:58.646Z12/30 Write block 504 len 8192 data: 10 8
215162024-02-08T06:47:58.646Z13/30 Flush
215172024-02-08T06:47:58.649Z14/30 Flush
215182024-02-08T06:47:58.649Z15/30 Write block 2 len 28672 data: 6 8 7 8 9 10 10
215192024-02-08T06:47:58.652Z16/30 Write block 442 len 36864 data: 14 15 15 14 11 12 14 13 11
215202024-02-08T06:47:58.652Z17/30 Write block 97 len 20480 data: 25 23 24 25 19
215212024-02-08T06:47:58.652Z18/30 Write block 193 len 8192 data: 12 14
215222024-02-08T06:47:58.652Z19/30 Read block 233 len 28672
215232024-02-08T06:47:58.655Z20/30 Read block 141 len 16384
215242024-02-08T06:47:58.655Z21/30 Flush
215252024-02-08T06:47:58.658Z22/30 Write block 63 len 40960 data: 11 9 10 9 9 14 15 10 11 13
215262024-02-08T06:47:58.658Z23/30 Read block 245 len 36864
215272024-02-08T06:47:58.661Z24/30 Write block 496 len 8192 data: 12 10
215282024-02-08T06:47:58.661Z25/30 Flush
215292024-02-08T06:47:58.664Z26/30 Write block 11 len 32768 data: 11 15 17 16 15 15 15 16
215302024-02-08T06:47:58.664Z27/30 Write block 360 len 36864 data: 15 11 13 13 16 17 16 19 16
215312024-02-08T06:47:58.664Z28/30 Read block 65 len 36864
215322024-02-08T06:47:58.667Z29/30 Write block 449 len 4096 data: 14
215332024-02-08T06:47:58.667Z30/30 Read block 41 len 16384
215342024-02-08T06:47:58.667Z----------------------------------------------------------------
215352024-02-08T06:47:58.667Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
215362024-02-08T06:47:58.667ZStates: Active Active Active
215372024-02-08T06:47:58.667ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
215382024-02-08T06:47:58.668Z 1 Acked 1000 Write 10 Done Sent Done false
215392024-02-08T06:47:58.668Z 2 Acked 1001 Write 3 Done Sent Done false
215402024-02-08T06:47:58.668Z 3 Acked 1002 Flush 0 Done Sent Done false
215412024-02-08T06:47:58.668Z 4 Acked 1003 Flush 0 Done Sent Done false
215422024-02-08T06:47:58.668Z 5 Acked 1004 Write 2 Done Sent Done false
215432024-02-08T06:47:58.668Z 6 Acked 1005 Write 10 Done Sent Done false
215442024-02-08T06:47:58.668Z 7 Acked 1006 Read 10 Done Sent Done false
215452024-02-08T06:47:58.668Z 8 Acked 1007 Read 5 Done Sent Done false
215462024-02-08T06:47:58.668Z 9 Acked 1008 Read 4 Done Sent Done false
215472024-02-08T06:47:58.668Z 10 Acked 1009 Read 10 Done Sent Done false
215482024-02-08T06:47:58.668Z 11 Acked 1010 Write 8 Done Sent Done false
215492024-02-08T06:47:58.668Z 12 Acked 1011 Write 2 Done Sent Done false
215502024-02-08T06:47:58.668Z 13 Acked 1012 Flush 0 Done Sent Done false
215512024-02-08T06:47:58.668Z 14 Acked 1013 Flush 0 Done Sent Done false
215522024-02-08T06:47:58.668Z 15 Acked 1014 Write 7 Done Sent Done false
215532024-02-08T06:47:58.668Z 16 Acked 1015 Write 9 Done Sent Done false
215542024-02-08T06:47:58.669Z 17 Acked 1016 Write 5 Done Sent Done false
215552024-02-08T06:47:58.669Z 18 Acked 1017 Write 2 Done Sent Done false
215562024-02-08T06:47:58.669Z 19 Acked 1018 Read 7 Done Sent Done false
215572024-02-08T06:47:58.669Z 20 Acked 1019 Read 4 Done Sent Done false
215582024-02-08T06:47:58.669Z 21 Acked 1020 Flush 0 Done Sent Done false
215592024-02-08T06:47:58.669Z 22 Acked 1021 Write 10 Done Sent Done false
215602024-02-08T06:47:58.669Z 23 Acked 1022 Read 9 Done Sent Done false
215612024-02-08T06:47:58.669Z 24 Acked 1023 Write 2 Done Sent Done false
215622024-02-08T06:47:58.669Z 25 Acked 1024 Flush 0 Done Sent Done false
215632024-02-08T06:47:58.669Z 26 Acked 1025 Write 8 Done Sent Done false
215642024-02-08T06:47:58.669Z 27 Acked 1026 Write 9 Done Sent Done false
215652024-02-08T06:47:58.669Z 28 Acked 1027 Read 9 Done Sent Done false
215662024-02-08T06:47:58.669Z 29 Acked 1028 Write 1 Done Sent Done false
215672024-02-08T06:47:58.669Z 30 Acked 1029 Read 4 Done Sent Done false
215682024-02-08T06:47:58.669Z STATES DS:0 DS:1 DS:2 TOTAL
215692024-02-08T06:47:58.670Z New 0 0 0 0
215702024-02-08T06:47:58.670Z Sent 0 30 0 30
215712024-02-08T06:47:58.670Z Done 30 0 30 60
215722024-02-08T06:47:58.670Z Skipped 0 0 0 0
215732024-02-08T06:47:58.670Z Error 0 0 0 0
215742024-02-08T06:47:58.670ZLast Flush: 1024 0 1024
215752024-02-08T06:47:58.670ZDownstairs last five completed:
215762024-02-08T06:47:58.670ZUpstairs last five completed: 30 29 28 27 26
215772024-02-08T06:47:58.670Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-08T06:47:57.87245556Z","hostname":"ip-10-150-1-117.us-west-2.compute.internal","pid":1930}
215782024-02-08T06:47:58.670ZWrote out file "/tmp/test_repair_verify.data"
215792024-02-08T06:47:58.673Z
215802024-02-08T06:47:58.673ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
215812024-02-08T06:47:58.689ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
215822024-02-08T06:47:58.689Z 0 000-019 198 196 198 343 337 343 T F T <---
215832024-02-08T06:47:58.689Z 1 020-039 196 196 196 337 337 337 F F F
215842024-02-08T06:47:58.689Z 2 040-059 197 197 197 338 338 338 F F F
215852024-02-08T06:47:58.692Z 3 060-079 198 192 198 344 327 344 F F F <---
215862024-02-08T06:47:58.692Z 4 080-099 198 197 198 343 338 343 F F F <---
215872024-02-08T06:47:58.692Z 5 100-119 198 197 198 343 338 343 F F F <---
215882024-02-08T06:47:58.692Z 6 120-139 194 194 194 332 332 332 F F F
215892024-02-08T06:47:58.692Z 7 140-159 194 194 194 332 332 332 F F F
215902024-02-08T06:47:58.693Z 8 160-179 183 183 183 312 312 312 F F F
215912024-02-08T06:47:58.693Z 9 180-199 198 197 198 343 338 343 F F F <---
215922024-02-08T06:47:58.693Z 10 200-219 198 197 198 341 338 341 F F F <---
215932024-02-08T06:47:58.693Z 11 220-239 198 194 198 341 332 341 F F F <---
215942024-02-08T06:47:58.693Z 12 240-259 197 197 197 338 338 338 F F F
215952024-02-08T06:47:58.693Z 13 260-279 197 197 197 338 338 338 F F F
215962024-02-08T06:47:58.693Z 14 280-299 190 190 190 326 326 326 F F F
215972024-02-08T06:47:58.693Z 15 300-319 194 194 194 332 332 332 F F F
215982024-02-08T06:47:58.693Z 16 320-339 198 194 198 339 332 339 F F F <---
215992024-02-08T06:47:58.693Z 17 340-359 194 194 194 332 332 332 F F F
216002024-02-08T06:47:58.693Z 18 360-379 197 197 197 338 338 338 T F T <---
216012024-02-08T06:47:58.693Z 19 380-399 185 185 185 317 317 317 F F F
216022024-02-08T06:47:58.693Z 20 400-419 197 197 197 338 338 338 F F F
216032024-02-08T06:47:58.693Z 21 420-439 192 192 192 330 330 330 F F F
216042024-02-08T06:47:58.693Z 22 440-459 198 188 198 343 324 343 T F T <---
216052024-02-08T06:47:58.694Z 23 460-479 197 197 197 338 338 338 F F F
216062024-02-08T06:47:58.694Z 24 480-499 198 197 198 344 338 344 F F F <---
216072024-02-08T06:47:58.694Z 25 500-519 198 188 198 341 320 341 F F F <---
216082024-02-08T06:47:58.694Z 26 520-539 198 194 198 341 332 341 F F F <---
216092024-02-08T06:47:58.694Z 27 540-559 198 194 198 341 332 341 F F F <---
216102024-02-08T06:47:58.694Z 28 560-579 192 192 192 327 327 327 F F F
216112024-02-08T06:47:58.694Z 29 580-599 198 184 198 339 315 339 F F F <---
216122024-02-08T06:47:58.694ZMax gen: 198, Max flush: 344
216132024-02-08T06:47:58.694ZError: Difference in extent metadata found!
216142024-02-08T06:47:58.694ZA Difference in extent metadata is expected here
216152024-02-08T06:47:58.694ZOn loop 99
216162024-02-08T06:47:58.694Z
216172024-02-08T06:47:58.698ZVerifying data now
216182024-02-08T06:48:03.414ZNo change after verify
216192024-02-08T06:48:03.414ZLoop: 99 Downstairs dump after verify (and repair):
216202024-02-08T06:48:03.427ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
216212024-02-08T06:48:03.427Z 0 000-019 199 199 199 345 345 345 F F F
216222024-02-08T06:48:03.427Z 1 020-039 196 196 196 337 337 337 F F F
216232024-02-08T06:48:03.427Z 2 040-059 197 197 197 338 338 338 F F F
216242024-02-08T06:48:03.427Z 3 060-079 198 198 198 344 344 344 F F F
216252024-02-08T06:48:03.427Z 4 080-099 198 198 198 343 343 343 F F F
216262024-02-08T06:48:03.445Z 5 100-119 198 198 198 343 343 343 F F F
216272024-02-08T06:48:03.446Z 6 120-139 194 194 194 332 332 332 F F F
216282024-02-08T06:48:03.446Z 7 140-159 194 194 194 332 332 332 F F F
216292024-02-08T06:48:03.446Z 8 160-179 183 183 183 312 312 312 F F F
216302024-02-08T06:48:03.446Z 9 180-199 198 198 198 343 343 343 F F F
216312024-02-08T06:48:03.446Z 10 200-219 198 198 198 341 341 341 F F F
216322024-02-08T06:48:03.446Z 11 220-239 198 198 198 341 341 341 F F F
216332024-02-08T06:48:03.446Z 12 240-259 197 197 197 338 338 338 F F F
216342024-02-08T06:48:03.446Z 13 260-279 197 197 197 338 338 338 F F F
216352024-02-08T06:48:03.447Z 14 280-299 190 190 190 326 326 326 F F F
216362024-02-08T06:48:03.447Z 15 300-319 194 194 194 332 332 332 F F F
216372024-02-08T06:48:03.447Z 16 320-339 198 198 198 339 339 339 F F F
216382024-02-08T06:48:03.447Z 17 340-359 194 194 194 332 332 332 F F F
216392024-02-08T06:48:03.447Z 18 360-379 199 199 199 345 345 345 F F F
216402024-02-08T06:48:03.447Z 19 380-399 185 185 185 317 317 317 F F F
216412024-02-08T06:48:03.447Z 20 400-419 197 197 197 338 338 338 F F F
216422024-02-08T06:48:03.447Z 21 420-439 192 192 192 330 330 330 F F F
216432024-02-08T06:48:03.447Z 22 440-459 199 199 199 345 345 345 F F F
216442024-02-08T06:48:03.448Z 23 460-479 197 197 197 338 338 338 F F F
216452024-02-08T06:48:03.448Z 24 480-499 198 198 198 344 344 344 F F F
216462024-02-08T06:48:03.448Z 25 500-519 198 198 198 341 341 341 F F F
216472024-02-08T06:48:03.448Z 26 520-539 198 198 198 341 341 341 F F F
216482024-02-08T06:48:03.448Z 27 540-559 198 198 198 341 341 341 F F F
216492024-02-08T06:48:03.448Z 28 560-579 192 192 192 327 327 327 F F F
216502024-02-08T06:48:03.448Z 29 580-599 198 198 198 339 339 339 F F F
216512024-02-08T06:48:03.448ZMax gen: 199, Max flush: 345
216522024-02-08T06:48:03.448Z8:55 Test duration
216532024-02-08T06:48:03.448ZTest completed
216542024-02-08T06:48:03.449Z
216552024-02-08T06:48:03.449Zreal 8:54.897533389
216562024-02-08T06:48:03.449Zuser 1:36.022612106
216572024-02-08T06:48:03.449Zsys 17.065979663
216582024-02-08T06:48:03.449Ztrap 0.086061845
216592024-02-08T06:48:03.449Ztflt 0.048731224
216602024-02-08T06:48:03.449Zdflt 0.045055774
216612024-02-08T06:48:03.449Zkflt 0.002915720
216622024-02-08T06:48:03.449Zlock 6:55:19.515726638
216632024-02-08T06:48:03.449Zslp 42:25.672028195
216642024-02-08T06:48:03.449Zlat 5.590600670
216652024-02-08T06:48:03.449Zstop 0.044919466
216662024-02-08T06:48:03.449Zprocess exited: duration 544027 ms, exit code 0
 
216672024-02-08T06:48:03.477Zfound 4 output files
216682024-02-08T06:48:03.477Zuploading: /tmp/test_repair_ds8810.txt (43767 bytes)
216692024-02-08T06:48:04.520Zuploaded: /tmp/test_repair_ds8810.txt
216702024-02-08T06:48:04.520Zuploading: /tmp/test_repair_ds8820.txt (23418 bytes)
216712024-02-08T06:48:05.548Zuploaded: /tmp/test_repair_ds8820.txt
216722024-02-08T06:48:05.548Zuploading: /tmp/test_repair_ds8830.txt (126494 bytes)
216732024-02-08T06:48:06.578Zuploaded: /tmp/test_repair_ds8830.txt
216742024-02-08T06:48:06.578Zuploading: /tmp/test_repair_out.txt (98417 bytes)
216752024-02-08T06:48:07.599Zuploaded: /tmp/test_repair_out.txt