01HKT6T8PN057WFDG75DPKNWFH: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HKT6TQG1TNYBFM40C266T292

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-01-10T17:51:53.923Zjob dependencies complete; ready to run (waiting for 20 m 56 s)
22024-01-10T17:53:15.241Zjob assigned to worker 01HKT814AH71BZP901GPNHSQZR (queued for 1 m 21 s)
32024-01-10T17:53:15.384Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-01-10T17:53:30.967Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-01-10T17:53:30.967Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-01-10T17:53:46.228Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-01-10T17:53:46.228Zdownloading input: /input/build/work/bins/crutest.gz
82024-01-10T17:54:07.098Zdownloaded input: /input/build/work/bins/crutest.gz
92024-01-10T17:54:07.098Zdownloading input: /input/build/work/bins/dsc.gz
102024-01-10T17:54:19.540Zdownloaded input: /input/build/work/bins/dsc.gz
112024-01-10T17:54:19.546Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-01-10T17:54:19.557Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-01-10T17:54:19.557Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-01-10T17:54:19.571Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-01-10T17:54:19.571Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-01-10T17:54:19.588Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-01-10T17:54:19.588Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-01-10T17:54:19.604Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-01-10T17:54:19.604Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-01-10T17:54:19.634Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-01-10T17:54:19.634Zdownloading input: /input/build/work/scripts/test_up.sh
222024-01-10T17:54:20.122Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-01-10T17:54:20.122Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-01-10T17:54:20.154Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-01-10T17:54:20.162Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-01-10T17:54:42.561Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-01-10T17:54:42.561Zstarting task 0: "setup"
282024-01-10T17:54:42.682Z++ uname -s
292024-01-10T17:54:42.685Z+ kern=SunOS
302024-01-10T17:54:42.685Z+ case "$kern" in
312024-01-10T17:54:42.685Z+ groupadd -g 12345 build
322024-01-10T17:54:42.685Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-01-10T17:54:44.722Z+ zfs create -o mountpoint=/work rpool/work
342024-01-10T17:54:44.772Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-01-10T17:54:44.775Z+ home_fs=zfs
362024-01-10T17:54:44.775Z+ [[ zfs == autofs ]]
372024-01-10T17:54:44.775Z+ mkdir -p /home/build
382024-01-10T17:54:44.778Z+ chown build:build /home/build /work
392024-01-10T17:54:46.834Z+ chmod 0700 /home/build /work
402024-01-10T17:54:46.843Zprocess exited: duration 4256 ms, exit code 0
 
412024-01-10T17:54:46.875Zstarting task 1: "authentication"
422024-01-10T17:54:46.897Zprocess exited: duration 28 ms, exit code 0
 
432024-01-10T17:54:46.906Zstarting task 2: "build"
442024-01-10T17:54:46.914Z+ banner cores
452024-01-10T17:54:46.951Z
462024-01-10T17:54:46.951Z #### #### ##### ###### ####
472024-01-10T17:54:46.951Z # # # # # # # #
482024-01-10T17:54:46.951Z # # # # # ##### ####
492024-01-10T17:54:46.952Z # # # ##### # #
502024-01-10T17:54:46.952Z # # # # # # # # #
512024-01-10T17:54:46.952Z #### #### # # ###### ####
522024-01-10T17:54:46.952Z
532024-01-10T17:54:46.962Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-01-10T17:54:46.966Z+ echo 'input bins dir contains:'
552024-01-10T17:54:46.966Zinput bins dir contains:
562024-01-10T17:54:46.966Z+ ls -ltr /input/build/work/bins
572024-01-10T17:54:46.969Ztotal 698594
582024-01-10T17:54:46.969Z-rw-r--r-- 1 root root 109456429 Jan 10 17:53 crucible-downstairs.gz
592024-01-10T17:54:46.969Z-rw-r--r-- 1 root root 85807289 Jan 10 17:53 crucible-hammer.gz
602024-01-10T17:54:46.969Z-rw-r--r-- 1 root root 93634688 Jan 10 17:54 crutest.gz
612024-01-10T17:54:46.969Z-rw-r--r-- 1 root root 68539139 Jan 10 17:54 dsc.gz
622024-01-10T17:54:46.969Z+ echo 'input script dir contains:'
632024-01-10T17:54:46.969Zinput script dir contains:
642024-01-10T17:54:46.969Z+ ls -ltr /input/build/work/scripts
652024-01-10T17:54:46.974Ztotal 38
662024-01-10T17:54:46.974Z-rw-r--r-- 1 root root 3056 Jan 10 17:54 perf-downstairs-tick.d
672024-01-10T17:54:46.974Z-rw-r--r-- 1 root root 939 Jan 10 17:54 test_ds.sh
682024-01-10T17:54:46.975Z-rw-r--r-- 1 root root 3746 Jan 10 17:54 test_live_repair.sh
692024-01-10T17:54:46.977Z-rw-r--r-- 1 root root 6416 Jan 10 17:54 test_repair.sh
702024-01-10T17:54:46.978Z-rw-r--r-- 1 root root 2969 Jan 10 17:54 test_replay.sh
712024-01-10T17:54:46.978Z-rw-r--r-- 1 root root 9465 Jan 10 17:54 test_up.sh
722024-01-10T17:54:46.978Z-rw-r--r-- 1 root root 2796 Jan 10 17:54 upstairs_info.d
732024-01-10T17:54:46.978Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-01-10T17:54:46.978Z+ echo ' chmod input script dir contains:'
752024-01-10T17:54:46.978Z chmod input script dir contains:
762024-01-10T17:54:46.978Z+ ls -ltr /input/build/work/scripts
772024-01-10T17:54:46.982Ztotal 38
782024-01-10T17:54:46.982Z-rwxr-xr-x 1 root root 3056 Jan 10 17:54 perf-downstairs-tick.d
792024-01-10T17:54:46.982Z-rwxr-xr-x 1 root root 939 Jan 10 17:54 test_ds.sh
802024-01-10T17:54:46.982Z-rwxr-xr-x 1 root root 3746 Jan 10 17:54 test_live_repair.sh
812024-01-10T17:54:46.982Z-rwxr-xr-x 1 root root 6416 Jan 10 17:54 test_repair.sh
822024-01-10T17:54:46.982Z-rwxr-xr-x 1 root root 2969 Jan 10 17:54 test_replay.sh
832024-01-10T17:54:46.982Z-rwxr-xr-x 1 root root 9465 Jan 10 17:54 test_up.sh
842024-01-10T17:54:46.983Z-rwxr-xr-x 1 root root 2796 Jan 10 17:54 upstairs_info.d
852024-01-10T17:54:46.983Z+ banner unpack
862024-01-10T17:54:46.983Z
872024-01-10T17:54:46.983Z # # # # ##### ## #### # #
882024-01-10T17:54:46.983Z # # ## # # # # # # # # #
892024-01-10T17:54:46.983Z # # # # # # # # # # ####
902024-01-10T17:54:46.983Z # # # # # ##### ###### # # #
912024-01-10T17:54:46.983Z # # # ## # # # # # # #
922024-01-10T17:54:46.983Z #### # # # # # #### # #
932024-01-10T17:54:46.983Z
942024-01-10T17:54:46.983Z+ mkdir -p /var/tmp/bins
952024-01-10T17:54:46.987Z+ for t in "$input/bins/"*.gz
962024-01-10T17:54:46.987Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-01-10T17:54:46.987Z+ b=crucible-downstairs.gz
982024-01-10T17:54:46.987Z+ b=crucible-downstairs
992024-01-10T17:54:46.987Z+ gunzip
1002024-01-10T17:54:50.798Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-01-10T17:54:50.803Z+ for t in "$input/bins/"*.gz
1022024-01-10T17:54:50.810Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-01-10T17:54:50.810Z+ b=crucible-hammer.gz
1042024-01-10T17:54:50.810Z+ b=crucible-hammer
1052024-01-10T17:54:50.810Z+ gunzip
1062024-01-10T17:54:53.841Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-01-10T17:54:53.851Z+ for t in "$input/bins/"*.gz
1082024-01-10T17:54:53.851Z++ basename /input/build/work/bins/crutest.gz
1092024-01-10T17:54:53.851Z+ b=crutest.gz
1102024-01-10T17:54:53.852Z+ b=crutest
1112024-01-10T17:54:53.852Z+ gunzip
1122024-01-10T17:54:57.077Z+ chmod +x /var/tmp/bins/crutest
1132024-01-10T17:54:57.101Z+ for t in "$input/bins/"*.gz
1142024-01-10T17:54:57.101Z++ basename /input/build/work/bins/dsc.gz
1152024-01-10T17:54:57.102Z+ b=dsc.gz
1162024-01-10T17:54:57.103Z+ b=dsc
1172024-01-10T17:54:57.103Z+ gunzip
1182024-01-10T17:54:59.814Z+ chmod +x /var/tmp/bins/dsc
1192024-01-10T17:54:59.823Z+ export BINDIR=/var/tmp/bins
1202024-01-10T17:54:59.823Z+ BINDIR=/var/tmp/bins
1212024-01-10T17:54:59.823Z+ banner repair
1222024-01-10T17:54:59.824Z
1232024-01-10T17:54:59.824Z ##### ###### ##### ## # #####
1242024-01-10T17:54:59.824Z # # # # # # # # # #
1252024-01-10T17:54:59.824Z # # ##### # # # # # # #
1262024-01-10T17:54:59.825Z ##### # ##### ###### # #####
1272024-01-10T17:54:59.825Z # # # # # # # # #
1282024-01-10T17:54:59.826Z # # ###### # # # # # #
1292024-01-10T17:54:59.826Z
1302024-01-10T17:54:59.826Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-01-10T17:54:59.826ZTurn off color for downstairs dump
1322024-01-10T17:54:59.826ZCreating region directory at: /var/tmp/test_repair
1332024-01-10T17:54:59.826ZCreating dsc directory at: /tmp/dsc
1342024-01-10T17:54:59.826ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.25173628
1352024-01-10T17:54:59.826ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.0915681
1362024-01-10T17:54:59.885ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.09702781
1372024-01-10T17:54:59.885ZRegion set with 3 regions was created
1382024-01-10T17:54:59.916Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1157 1158 1159
1392024-01-10T17:54:59.916ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-01-10T17:55:00.044Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.850226249Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161}
1412024-01-10T17:55:00.045Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.854814229Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161}
1422024-01-10T17:55:00.045Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.854976195Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161}
1432024-01-10T17:55:00.046Z{"msg":"Crucible 26990838-04b5-4fd3-936e-c2ff4bd993e1 has session id: 84e6a489-67cb-4e77-a34b-58e28dad3956","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.855062093Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1442024-01-10T17:55:00.046Z{"msg":"Upstairs opts: Upstairs UUID: 26990838-04b5-4fd3-936e-c2ff4bd993e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.855225988Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1452024-01-10T17:55:00.046Z{"msg":"Crucible stats registered with UUID: 26990838-04b5-4fd3-936e-c2ff4bd993e1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.855314369Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1462024-01-10T17:55:00.066ZCrucible runtime is spawned
1472024-01-10T17:55:00.066Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.85774208Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161}
1482024-01-10T17:55:00.066Z{"msg":"26990838-04b5-4fd3-936e-c2ff4bd993e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.858201932Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1492024-01-10T17:55:00.066Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.858288487Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1502024-01-10T17:55:00.066Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.858468713Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1512024-01-10T17:55:00.066Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.858577705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1522024-01-10T17:55:00.067Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.858681732Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1532024-01-10T17:55:00.068Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.858781187Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1542024-01-10T17:55:00.068Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.85888182Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1552024-01-10T17:55:00.068Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.859342698Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1562024-01-10T17:55:00.068Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.859991617Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1572024-01-10T17:55:00.068Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.860086709Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1582024-01-10T17:55:00.074Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.862532226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1592024-01-10T17:55:00.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.86262634Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1602024-01-10T17:55:00.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.862933829Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1612024-01-10T17:55:00.075Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.866143683Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1622024-01-10T17:55:00.075Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.86625087Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1632024-01-10T17:55:00.075Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.866316405Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1642024-01-10T17:55:00.075Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.866360547Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1652024-01-10T17:55:00.075Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.866461271Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1662024-01-10T17:55:00.075Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.866544334Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1672024-01-10T17:55:00.075Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.869344776Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1682024-01-10T17:55:00.075Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.869577441Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1692024-01-10T17:55:00.076Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.869701343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1702024-01-10T17:55:00.076Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.870899797Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1712024-01-10T17:55:00.076Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.871002676Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1722024-01-10T17:55:00.076Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.871101129Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1732024-01-10T17:55:00.076Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.871145599Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1742024-01-10T17:55:00.076Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.87138837Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1752024-01-10T17:55:00.076Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.871630909Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1762024-01-10T17:55:00.076Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.871710936Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1772024-01-10T17:55:00.076Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.871755151Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1782024-01-10T17:55:00.076Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.872220643Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1792024-01-10T17:55:00.077Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.872286342Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1802024-01-10T17:55:00.077Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873196022Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1812024-01-10T17:55:00.078Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873248979Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1822024-01-10T17:55:00.078Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873274343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1832024-01-10T17:55:00.078Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873316546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1842024-01-10T17:55:00.079Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873340534Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1852024-01-10T17:55:00.079Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873381034Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1862024-01-10T17:55:00.079Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873405704Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1872024-01-10T17:55:00.079Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873444925Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1882024-01-10T17:55:00.079Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873467699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1892024-01-10T17:55:00.079Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873506434Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1902024-01-10T17:55:00.079Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873530597Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1912024-01-10T17:55:00.079Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873571401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1922024-01-10T17:55:00.079Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873593018Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1932024-01-10T17:55:00.080Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873614973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1942024-01-10T17:55:00.080Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.87363664Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1952024-01-10T17:55:00.080Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1962024-01-10T17:55:00.080Z,"time":"2024-01-10T17:54:59.873676637Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1972024-01-10T17:55:00.081Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873699632Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1982024-01-10T17:55:00.081Z{"msg":"26990838-04b5-4fd3-936e-c2ff4bd993e1 is now active with session: 84e6a489-67cb-4e77-a34b-58e28dad3956","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873721069Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
1992024-01-10T17:55:00.081Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873757922Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161,"session_id":"84e6a489-67cb-4e77-a34b-58e28dad3956"}
2002024-01-10T17:55:00.081Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:54:59.873779195Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1161}
2012024-01-10T17:55:00.081ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-01-10T17:55:00.081ZFill test
2032024-01-10T17:55:00.178ZRead and Verify all blocks (0..600 range:false)
2042024-01-10T17:55:00.421ZWrote out file "/tmp/test_repair_verify.data"
2052024-01-10T17:55:00.421ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-01-10T17:55:00.426Z----------------------------------------------------------------
2072024-01-10T17:55:00.426Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-01-10T17:55:00.426ZStates: Active Active Active
2092024-01-10T17:55:00.426ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-01-10T17:55:00.426Z 8 Acked 1007 Read 100 Done Done Done false
2112024-01-10T17:55:00.426Z 9 Acked 1008 Read 100 Done Done Done false
2122024-01-10T17:55:00.426Z 10 Acked 1009 Read 100 Done Done Done false
2132024-01-10T17:55:00.426Z 11 Acked 1010 Read 100 Done Done Done false
2142024-01-10T17:55:00.426Z 12 Acked 1011 Read 100 Done Done Done false
2152024-01-10T17:55:00.427Z 13 Acked 1012 Read 100 Done Done Done false
2162024-01-10T17:55:00.427Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-01-10T17:55:00.427Z New 0 0 0 0
2182024-01-10T17:55:00.427Z Sent 0 0 0 0
2192024-01-10T17:55:00.427Z Done 6 6 6 18
2202024-01-10T17:55:00.427Z Skipped 0 0 0 0
2212024-01-10T17:55:00.427Z Error 0 0 0 0
2222024-01-10T17:55:00.427ZLast Flush: 1006 1006 1006
2232024-01-10T17:55:00.431ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-01-10T17:55:00.432ZUpstairs last five completed: 13 12 11 10 9
2252024-01-10T17:55:00.432ZCLIENT: Up:0 ds:6 act:3
2262024-01-10T17:55:04.414Z----------------------------------------------------------------
2272024-01-10T17:55:04.415Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-01-10T17:55:04.415ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-01-10T17:55:04.415ZUpstairs last five completed: 14 13 12 11 10
2302024-01-10T17:55:04.421ZCLIENT: Up:0 ds:0 act:3
2312024-01-10T17:55:04.421ZCLIENT: All crucible jobs finished, exiting program
2322024-01-10T17:55:04.424Z
2332024-01-10T17:55:04.424ZBegin loop 1
2342024-01-10T17:55:04.424ZDownstairs to restart: 2
2352024-01-10T17:55:04.449Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.260237133Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163}
2362024-01-10T17:55:04.454Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.260789671Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163}
2372024-01-10T17:55:04.454Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.260944207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163}
2382024-01-10T17:55:04.454Z{"msg":"Crucible 388ad678-7ae8-4e89-b43b-15e97436dd73 has session id: c67f8946-7254-4e09-8ffb-9d2c4d7f2392","v":0,"name":"crucible","level":30Crucible runtime is spawned
2392024-01-10T17:55:04.454Z,"time":"2024-01-10T17:55:04.261030108Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2402024-01-10T17:55:04.454Z{"msg":"Upstairs opts: Upstairs UUID: 388ad678-7ae8-4e89-b43b-15e97436dd73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261122258Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2412024-01-10T17:55:04.457Z{"msg":"Crucible stats registered with UUID: 388ad678-7ae8-4e89-b43b-15e97436dd73","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261169012Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2422024-01-10T17:55:04.457Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261208723Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163}
2432024-01-10T17:55:04.457Z{"msg":"388ad678-7ae8-4e89-b43b-15e97436dd73 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261372385Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2442024-01-10T17:55:04.457Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261450256Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"0","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2452024-01-10T17:55:04.457Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261552528Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"0","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2462024-01-10T17:55:04.457Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261599708Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"1","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2472024-01-10T17:55:04.457Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261668746Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"1","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2482024-01-10T17:55:04.457Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261697445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"2","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2492024-01-10T17:55:04.457Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261733348Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"2","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2502024-01-10T17:55:04.457Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261800352Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"io task","client":"1","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2512024-01-10T17:55:04.458Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261849723Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"io task","client":"0","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2522024-01-10T17:55:04.458Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261896733Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"io task","client":"2","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2532024-01-10T17:55:04.458Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261943129Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"io task","client":"2","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2542024-01-10T17:55:04.458Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.261984114Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"io task","client":"1","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2552024-01-10T17:55:04.458Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.262075433Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"io task","client":"0","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2562024-01-10T17:55:04.458Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.263368659Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"0","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2572024-01-10T17:55:04.458Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.263453801Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"0","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2582024-01-10T17:55:04.458Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.263600139Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"1","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2592024-01-10T17:55:04.458Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.263644671Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"1","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2602024-01-10T17:55:04.458Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.264394025Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"0","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2612024-01-10T17:55:04.458Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.264478158Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"1","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2622024-01-10T17:55:04.463Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.264764679Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"2","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2632024-01-10T17:55:04.463Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.26487274Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"2","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2642024-01-10T17:55:04.463Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.26539051Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"1","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2652024-01-10T17:55:04.463Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.265450905Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"1","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2662024-01-10T17:55:04.463Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.265477038Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2672024-01-10T17:55:04.463Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.26562601Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"0","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2682024-01-10T17:55:04.463Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.265683437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"0","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2692024-01-10T17:55:04.464Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.265707984Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2702024-01-10T17:55:04.464Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.266094438Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"2","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2712024-01-10T17:55:04.465Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.266814381Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"2","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2722024-01-10T17:55:04.466Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.266899839Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"2","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2732024-01-10T17:55:04.466Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267008728Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2742024-01-10T17:55:04.466Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267061691Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2752024-01-10T17:55:04.466ZWait for a query_work_queue command to finish before sending IO
2762024-01-10T17:55:04.466Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267094107Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2772024-01-10T17:55:04.466Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267135953Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2782024-01-10T17:55:04.466Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267160005Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2792024-01-10T17:55:04.466Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267185264Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2802024-01-10T17:55:04.466Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267225831Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2812024-01-10T17:55:04.467Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267249651Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2822024-01-10T17:55:04.467Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267291031Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2832024-01-10T17:55:04.467Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267313678Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2842024-01-10T17:55:04.467Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267356931Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2852024-01-10T17:55:04.467Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267379153Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2862024-01-10T17:55:04.468Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267426607Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"":"downstairs","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2872024-01-10T17:55:04.468Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267449649Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2882024-01-10T17:55:04.468Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267494356Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"0","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2892024-01-10T17:55:04.468ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2902024-01-10T17:55:04.468Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.26752134Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"1","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2912024-01-10T17:55:04.468Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267561479Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"client":"2","":"downstairs","session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2922024-01-10T17:55:04.469Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267584498Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2932024-01-10T17:55:04.469Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.26762251Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2942024-01-10T17:55:04.469Z{"msg":"388ad678-7ae8-4e89-b43b-15e97436dd73 is now active with session: c67f8946-7254-4e09-8ffb-9d2c4d7f2392","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267644986Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2952024-01-10T17:55:04.469Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267683022Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163,"session_id":"c67f8946-7254-4e09-8ffb-9d2c4d7f2392"}
2962024-01-10T17:55:04.469Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:04.267704346Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1163}
2972024-01-10T17:55:04.469ZLoading write count information from file "/tmp/test_repair_verify.data"
2982024-01-10T17:55:04.469ZRun Repair workload
2992024-01-10T17:55:04.469Z01/30 Write block 411 len 16384 data: 2 2 2 2
3002024-01-10T17:55:04.469Z02/30 Flush
3012024-01-10T17:55:04.474Z03/30 Flush
3022024-01-10T17:55:04.474Z04/30 Write block 29 len 32768 data: 2 2 2 2 2 2 2 2
3032024-01-10T17:55:04.474Z05/30 Write block 473 len 8192 data: 2 2
3042024-01-10T17:55:04.474Z06/30 Write block 521 len 4096 data: 2
3052024-01-10T17:55:04.474Z07/30 Read block 177 len 36864
3062024-01-10T17:55:04.474Z08/30 Read block 410 len 16384
3072024-01-10T17:55:04.474Z09/30 Read block 89 len 8192
3082024-01-10T17:55:04.478Z10/30 Read block 129 len 12288
3092024-01-10T17:55:04.478Z11/30 Write block 322 len 4096 data: 2
3102024-01-10T17:55:04.479Z12/30 Write block 380 len 20480 data: 2 2 2 2 2
3112024-01-10T17:55:04.479Z13/30 Write block 54 len 20480 data: 2 2 2 2 2
3122024-01-10T17:55:04.482Z14/30 Read block 197 len 8192
3132024-01-10T17:55:04.482Z15/30 Read block 563 len 8192
3142024-01-10T17:55:04.482Z16/30 Write block 544 len 36864 data: 2 2 2 2 2 2 2 2 2
3152024-01-10T17:55:04.482Z17/30 Read block 287 len 24576
3162024-01-10T17:55:04.486Z18/30 Read block 11 len 36864
3172024-01-10T17:55:04.490Z19/30 Read block 237 len 32768
3182024-01-10T17:55:04.493Z20/30 Write block 582 len 12288 data: 2 2 2
3192024-01-10T17:55:04.493Z21/30 Flush
3202024-01-10T17:55:04.496Z22/30 Write block 147 len 16384 data: 2 2 2 2
3212024-01-10T17:55:04.499Z23/30 Read block 325 len 4096
3222024-01-10T17:55:04.499Z24/30 Read block 74 len 32768
3232024-01-10T17:55:04.502Z25/30 Read block 455 len 20480
3242024-01-10T17:55:04.503Z26/30 Read block 106 len 20480
3252024-01-10T17:55:04.506Z27/30 Read block 333 len 28672
3262024-01-10T17:55:04.510Z28/30 Read block 193 len 12288
3272024-01-10T17:55:04.510Z29/30 Read block 385 len 32768
3282024-01-10T17:55:04.513Z30/30 Read block 505 len 28672
3292024-01-10T17:55:04.517Z----------------------------------------------------------------
3302024-01-10T17:55:04.517Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:27
3312024-01-10T17:55:04.517ZStates: Active Active Active
3322024-01-10T17:55:04.517ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3332024-01-10T17:55:04.517Z 4 Acked 1003 Write 8 Done Done Done false
3342024-01-10T17:55:04.517Z 5 Acked 1004 Write 2 Done Done Done false
3352024-01-10T17:55:04.517Z 6 Acked 1005 Write 1 Done Done Done false
3362024-01-10T17:55:04.518Z 7 Acked 1006 Read 9 Done Done Done false
3372024-01-10T17:55:04.518Z 8 Acked 1007 Read 4 Done Done Done false
3382024-01-10T17:55:04.518Z 9 Acked 1008 Read 2 Done Done Done false
3392024-01-10T17:55:04.518Z 10 Acked 1009 Read 3 Done Done Done false
3402024-01-10T17:55:04.518Z 11 Acked 1010 Write 1 Done Done Done false
3412024-01-10T17:55:04.518Z 12 Acked 1011 Write 5 Done Done Sent false
3422024-01-10T17:55:04.518Z 13 Acked 1012 Write 5 Done Done Sent false
3432024-01-10T17:55:04.518Z 14 Acked 1013 Read 2 Done Done Sent false
3442024-01-10T17:55:04.518Z 15 Acked 1014 Read 2 Done Done Sent false
3452024-01-10T17:55:04.518Z 16 Acked 1015 Write 9 Done Done Sent false
3462024-01-10T17:55:04.518Z 17 Acked 1016 Read 6 Done Done Sent false
3472024-01-10T17:55:04.518Z 18 Acked 1017 Read 9 Done Done Sent false
3482024-01-10T17:55:04.518Z 19 Acked 1018 Read 8 Done Done Sent false
3492024-01-10T17:55:04.518Z 20 Acked 1019 Write 3 Done Done Sent false
3502024-01-10T17:55:04.518Z 21 Acked 1020 Flush 0 Done Done Sent false
3512024-01-10T17:55:04.519Z 22 Acked 1021 Write 4 Done Done Sent false
3522024-01-10T17:55:04.519Z 23 Acked 1022 Read 1 Done Done Sent false
3532024-01-10T17:55:04.519Z 24 Acked 1023 Read 8 Done Done Sent false
3542024-01-10T17:55:04.519Z 25 Acked 1024 Read 5 Done Done Sent false
3552024-01-10T17:55:04.519Z 26 Acked 1025 Read 5 Done Done Sent false
3562024-01-10T17:55:04.519Z 27 Acked 1026 Read 7 Done Done Sent false
3572024-01-10T17:55:04.519Z 28 Acked 1027 Read 3 Done Done Sent false
3582024-01-10T17:55:04.519Z 29 Acked 1028 Read 8 Done Done Sent false
3592024-01-10T17:55:04.519Z 30 Acked 1029 Read 7 Done Sent Sent false
3602024-01-10T17:55:04.519Z STATES DS:0 DS:1 DS:2 TOTAL
3612024-01-10T17:55:04.519Z New 0 0 0 0
3622024-01-10T17:55:04.519Z Sent 0 1 19 20
3632024-01-10T17:55:04.519Z Done 27 26 8 61
3642024-01-10T17:55:04.519Z Skipped 0 0 0 0
3652024-01-10T17:55:04.519Z Error 0 0 0 0
3662024-01-10T17:55:04.519ZLast Flush: 1020 1020 1002
3672024-01-10T17:55:04.519ZDownstairs last five completed: 1002 1001 1000
3682024-01-10T17:55:04.519ZUpstairs last five completed: 30 29 28 27 26
3692024-01-10T17:55:04.519ZWrote out file "/tmp/test_repair_verify.data"
3702024-01-10T17:55:04.523Z
3712024-01-10T17:55:04.529ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3722024-01-10T17:55:04.563ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3732024-01-10T17:55:04.573Z 0 000-019 1 1 1 1 1 1 F F F
3742024-01-10T17:55:04.573Z 1 020-039 2 2 1 4 4 1 F F T <---
3752024-01-10T17:55:04.573Z 2 040-059 2 2 1 4 4 1 F F F <---
3762024-01-10T17:55:04.573Z 3 060-079 1 1 1 1 1 1 F F F
3772024-01-10T17:55:04.573Z 4 080-099 1 1 1 1 1 1 F F F
3782024-01-10T17:55:04.573Z 5 100-119 1 1 1 1 1 1 F F F
3792024-01-10T17:55:04.573Z 6 120-139 1 1 1 1 1 1 F F F
3802024-01-10T17:55:04.574Z 7 140-159 1 1 1 1 1 1 T T F <---
3812024-01-10T17:55:04.574Z 8 160-179 1 1 1 1 1 1 F F F
3822024-01-10T17:55:04.574Z 9 180-199 1 1 1 1 1 1 F F F
3832024-01-10T17:55:04.574Z 10 200-219 1 1 1 1 1 1 F F F
3842024-01-10T17:55:04.576Z 11 220-239 1 1 1 1 1 1 F F F
3852024-01-10T17:55:04.576Z 12 240-259 1 1 1 1 1 1 F F F
3862024-01-10T17:55:04.576Z 13 260-279 1 1 1 1 1 1 F F F
3872024-01-10T17:55:04.576Z 14 280-299 1 1 1 1 1 1 F F F
3882024-01-10T17:55:04.576Z 15 300-319 1 1 1 1 1 1 F F F
3892024-01-10T17:55:04.576Z 16 320-339 2 2 1 4 4 1 F F T <---
3902024-01-10T17:55:04.576Z 17 340-359 1 1 1 1 1 1 F F F
3912024-01-10T17:55:04.576Z 18 360-379 1 1 1 1 1 1 F F F
3922024-01-10T17:55:04.576Z 19 380-399 2 2 1 4 4 1 F F F <---
3932024-01-10T17:55:04.576Z 20 400-419 2 2 2 2 2 2 F F F
3942024-01-10T17:55:04.576Z 21 420-439 1 1 1 1 1 1 F F F
3952024-01-10T17:55:04.576Z 22 440-459 1 1 1 1 1 1 F F F
3962024-01-10T17:55:04.576Z 23 460-479 2 2 1 4 4 1 F F T <---
3972024-01-10T17:55:04.576Z 24 480-499 1 1 1 1 1 1 F F F
3982024-01-10T17:55:04.576Z 25 500-519 1 1 1 1 1 1 F F F
3992024-01-10T17:55:04.576Z 26 520-539 2 2 1 4 4 1 F F T <---
4002024-01-10T17:55:04.576Z 27 540-559 2 2 1 4 4 1 F F F <---
4012024-01-10T17:55:04.577Z 28 560-579 1 1 1 1 1 1 F F F
4022024-01-10T17:55:04.577Z 29 580-599 2 2 1 4 4 1 F F F <---
4032024-01-10T17:55:04.577ZMax gen: 2, Max flush: 4
4042024-01-10T17:55:04.577ZError: Difference in extent metadata found!
4052024-01-10T17:55:04.577ZA Difference in extent metadata is expected here
4062024-01-10T17:55:04.577ZOn loop 1
4072024-01-10T17:55:04.577Z
4082024-01-10T17:55:04.577ZVerifying data now
4092024-01-10T17:55:10.538ZNo change after verify
4102024-01-10T17:55:10.538ZLoop: 1 Downstairs dump after verify (and repair):
4112024-01-10T17:55:10.963ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4122024-01-10T17:55:10.964Z 0 000-019 1 1 1 1 1 1 F F F
4132024-01-10T17:55:10.964Z 1 020-039 2 2 2 4 4 4 F F F
4142024-01-10T17:55:10.964Z 2 040-059 2 2 2 4 4 4 F F F
4152024-01-10T17:55:10.964Z 3 060-079 1 1 1 1 1 1 F F F
4162024-01-10T17:55:10.964Z 4 080-099 1 1 1 1 1 1 F F F
4172024-01-10T17:55:10.964Z 5 100-119 1 1 1 1 1 1 F F F
4182024-01-10T17:55:10.964Z 6 120-139 1 1 1 1 1 1 F F F
4192024-01-10T17:55:10.985Z 7 140-159 3 3 3 5 5 5 F F F
4202024-01-10T17:55:10.985Z 8 160-179 1 1 1 1 1 1 F F F
4212024-01-10T17:55:10.985Z 9 180-199 1 1 1 1 1 1 F F F
4222024-01-10T17:55:10.985Z 10 200-219 1 1 1 1 1 1 F F F
4232024-01-10T17:55:10.985Z 11 220-239 1 1 1 1 1 1 F F F
4242024-01-10T17:55:10.986Z 12 240-259 1 1 1 1 1 1 F F F
4252024-01-10T17:55:10.986Z 13 260-279 1 1 1 1 1 1 F F F
4262024-01-10T17:55:10.986Z 14 280-299 1 1 1 1 1 1 F F F
4272024-01-10T17:55:10.986Z 15 300-319 1 1 1 1 1 1 F F F
4282024-01-10T17:55:10.986Z 16 320-339 2 2 2 4 4 4 F F F
4292024-01-10T17:55:10.986Z 17 340-359 1 1 1 1 1 1 F F F
4302024-01-10T17:55:10.987Z 18 360-379 1 1 1 1 1 1 F F F
4312024-01-10T17:55:10.987Z 19 380-399 2 2 2 4 4 4 F F F
4322024-01-10T17:55:10.987Z 20 400-419 2 2 2 2 2 2 F F F
4332024-01-10T17:55:10.987Z 21 420-439 1 1 1 1 1 1 F F F
4342024-01-10T17:55:10.987Z 22 440-459 1 1 1 1 1 1 F F F
4352024-01-10T17:55:10.987Z 23 460-479 2 2 2 4 4 4 F F F
4362024-01-10T17:55:10.988Z 24 480-499 1 1 1 1 1 1 F F F
4372024-01-10T17:55:10.988Z 25 500-519 1 1 1 1 1 1 F F F
4382024-01-10T17:55:10.988Z 26 520-539 2 2 2 4 4 4 F F F
4392024-01-10T17:55:10.988Z 27 540-559 2 2 2 4 4 4 F F F
4402024-01-10T17:55:10.988Z 28 560-579 1 1 1 1 1 1 F F F
4412024-01-10T17:55:10.988Z 29 580-599 2 2 2 4 4 4 F F F
4422024-01-10T17:55:10.988ZMax gen: 3, Max flush: 5
4432024-01-10T17:55:10.988Z
4442024-01-10T17:55:10.988ZBegin loop 2
4452024-01-10T17:55:10.988ZDownstairs to restart: 1
4462024-01-10T17:55:10.988Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.400939343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171}
4472024-01-10T17:55:10.988Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.401440854Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171}
4482024-01-10T17:55:10.988Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.401522205Z","hostname":"Crucible runtime is spawned
4492024-01-10T17:55:10.989Zip-10-150-1-7.us-west-2.compute.internal","pid":1171}
4502024-01-10T17:55:10.989Z{"msg":"Crucible b2c769ce-586a-48a1-9762-78030d9b72ca has session id: b807d2d2-b44c-42a8-9485-5e2ecc63c33d","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.401569931Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4512024-01-10T17:55:10.990Z{"msg":"Upstairs opts: Upstairs UUID: b2c769ce-586a-48a1-9762-78030d9b72ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.401625311Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4522024-01-10T17:55:10.990Z{"msg":"Crucible stats registered with UUID: b2c769ce-586a-48a1-9762-78030d9b72ca","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.401713962Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4532024-01-10T17:55:10.990Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.401756234Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171}
4542024-01-10T17:55:10.990Z{"msg":"b2c769ce-586a-48a1-9762-78030d9b72ca active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.401997048Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4552024-01-10T17:55:10.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402119869Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"0","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4562024-01-10T17:55:10.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402199632Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"0","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4572024-01-10T17:55:10.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402274722Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"1","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4582024-01-10T17:55:10.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402554408Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"1","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4592024-01-10T17:55:10.991Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402633972Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"2","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4602024-01-10T17:55:10.991Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402785036Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"2","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4612024-01-10T17:55:10.991Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402817046Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"io task","client":"2","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4622024-01-10T17:55:10.992Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402860904Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"io task","client":"1","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4632024-01-10T17:55:10.992Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402885651Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"io task","client":"0","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4642024-01-10T17:55:10.992Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402908109Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"io task","client":"0","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4652024-01-10T17:55:10.992Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402930211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"io task","client":"1","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4662024-01-10T17:55:10.992Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.402952277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"io task","client":"2","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4672024-01-10T17:55:10.992Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.404015074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"2","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4682024-01-10T17:55:10.992Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.404134895Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"2","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4692024-01-10T17:55:10.992Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.40418123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"0","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4702024-01-10T17:55:10.992Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.404260976Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"0","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4712024-01-10T17:55:10.994Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.404308958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"1","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4722024-01-10T17:55:10.994Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.404384194Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"1","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4732024-01-10T17:55:10.994Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.405013117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"2","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4742024-01-10T17:55:10.994Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.405096087Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"0","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4752024-01-10T17:55:10.994Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.405266689Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"1","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4762024-01-10T17:55:10.994Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.405926479Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"2","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4772024-01-10T17:55:10.994Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406026507Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"2","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4782024-01-10T17:55:10.995Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.40613549Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4792024-01-10T17:55:10.995Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.40617743Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"0","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4802024-01-10T17:55:10.995Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406209732Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"0","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4812024-01-10T17:55:10.996Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406232811Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4822024-01-10T17:55:10.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406274795Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"1","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4832024-01-10T17:55:10.997Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406298918Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"1","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4842024-01-10T17:55:10.997Z{"msg":"[0]R flush_numbers[0..12]: [1, 4, 4, 1, 1, 1, 1, 5, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406354292Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4852024-01-10T17:55:10.997Z{"msg":"[0]R generation[0..12]: [1, 2, 2, 1, 1, 1, 1, 3, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.40639913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4862024-01-10T17:55:10.997Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406424508Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4872024-01-10T17:55:10.997Z{"msg":"[1]R flush_numbers[0..12]: [1, 4, 4, 1, 1, 1, 1, 5, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406465298Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4882024-01-10T17:55:10.997Z{"msg":"[1]R generation[0..12]: [1, 2, 2, 1, 1, 1, 1, 3, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406488861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4892024-01-10T17:55:10.999Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406529942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4902024-01-10T17:55:10.999Z{"msg":"[2]R flush_numbers[0..12]: [1, 4, 4, 1, 1, 1, 1, 5, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406553637Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4912024-01-10T17:55:11.000Z{"msg":"[2]R generation[0..12]: [1, 2, 2, 1, 1, 1, 1, 3, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.40659361Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","Wait for a query_work_queue command to finish before sending IO
4922024-01-10T17:55:11.001Zpid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4932024-01-10T17:55:11.001Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.40662223Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4942024-01-10T17:55:11.001Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406661626Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4952024-01-10T17:55:11.001Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406684802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4962024-01-10T17:55:11.002Z{"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406723802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4972024-01-10T17:55:11.002Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406746131Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"":"downstairs","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4982024-01-10T17:55:11.002Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406785659Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
4992024-01-10T17:55:11.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406806483Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"0","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
5002024-01-10T17:55:11.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406827995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"1","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
5012024-01-10T17:55:11.003Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406849149Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"client":"2","":"downstairs","session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
5022024-01-10T17:55:11.004Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406887859Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
5032024-01-10T17:55:11.004Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406908648Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
5042024-01-10T17:55:11.004Z{"msg":"b2c769ce-586a-48a1-9762-78030d9b72ca is now active with session: b807d2d2-b44c-42a8-9485-5e2ecc63c33d","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406930099Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
5052024-01-10T17:55:11.004Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406975177Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171,"session_id":"b807d2d2-b44c-42a8-9485-5e2ecc63c33d"}
5062024-01-10T17:55:11.004Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:10.406996385Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1171}
5072024-01-10T17:55:11.004ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5082024-01-10T17:55:11.004ZLoading write count information from file "/tmp/test_repair_verify.data"
5092024-01-10T17:55:11.004ZRun Repair workload
5102024-01-10T17:55:11.004Z01/30 Write block 21 len 40960 data: 2 2 2 2 2 2 2 2 3 3
5112024-01-10T17:55:11.004Z02/30 Read block 72 len 28672
5122024-01-10T17:55:11.004Z03/30 Write block 249 len 20480 data: 2 2 2 2 2
5132024-01-10T17:55:11.004Z04/30 Read block 258 len 40960
5142024-01-10T17:55:11.005Z05/30 Write block 267 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5152024-01-10T17:55:11.005Z06/30 Write block 419 len 16384 data: 2 2 2 2
5162024-01-10T17:55:11.005Z07/30 Read block 228 len 20480
5172024-01-10T17:55:11.005Z08/30 Write block 513 len 20480 data: 2 2 2 2 2
5182024-01-10T17:55:11.005Z09/30 Read block 544 len 36864
5192024-01-10T17:55:11.015Z10/30 Read block 27 len 40960
5202024-01-10T17:55:11.015Z11/30 Write block 184 len 28672 data: 2 2 2 2 2 2 2
5212024-01-10T17:55:11.015Z12/30 Write block 462 len 28672 data: 2 2 2 2 2 2 2
5222024-01-10T17:55:11.015Z13/30 Flush
5232024-01-10T17:55:11.015Z14/30 Write block 415 len 28672 data: 2 2 2 2 3 3 3
5242024-01-10T17:55:11.015Z15/30 Write block 445 len 36864 data: 2 2 2 2 2 2 2 2 2
5252024-01-10T17:55:11.015Z16/30 Read block 466 len 16384
5262024-01-10T17:55:11.015Z17/30 Read block 4 len 12288
5272024-01-10T17:55:11.015Z18/30 Write block 12 len 28672 data: 2 2 2 2 2 2 2
5282024-01-10T17:55:11.016Z19/30 Read block 551 len 28672
5292024-01-10T17:55:11.016Z20/30 Read block 518 len 12288
5302024-01-10T17:55:11.016Z21/30 Read block 356 len 24576
5312024-01-10T17:55:11.016Z22/30 Read block 350 len 32768
5322024-01-10T17:55:11.016Z23/30 Write block 507 len 28672 data: 2 2 2 2 2 2 3
5332024-01-10T17:55:11.016Z24/30 Read block 495 len 24576
5342024-01-10T17:55:11.016Z25/30 Write block 162 len 16384 data: 2 2 2 2
5352024-01-10T17:55:11.016Z26/30 Read block 528 len 16384
5362024-01-10T17:55:11.016Z27/30 Write block 495 len 36864 data: 2 2 2 2 2 2 2 2 2
5372024-01-10T17:55:11.017Z28/30 Read block 278 len 20480
5382024-01-10T17:55:11.017Z29/30 Write block 584 len 12288 data: 3 2 2
5392024-01-10T17:55:11.017Z30/30 Read block 498 len 24576
5402024-01-10T17:55:11.017Z----------------------------------------------------------------
5412024-01-10T17:55:11.017Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
5422024-01-10T17:55:11.017ZStates: Active Active Active
5432024-01-10T17:55:11.017ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5442024-01-10T17:55:11.018Z 1 Acked 1000 Write 10 Done Done Done false
5452024-01-10T17:55:11.018Z 2 Acked 1001 Read 7 Done Done Done false
5462024-01-10T17:55:11.018Z 3 Acked 1002 Write 5 Done Sent Done false
5472024-01-10T17:55:11.018Z 4 Acked 1003 Read 10 Done Sent Done false
5482024-01-10T17:55:11.018Z 5 Acked 1004 Write 10 Done Sent Done false
5492024-01-10T17:55:11.018Z 6 Acked 1005 Write 4 Done Sent Done false
5502024-01-10T17:55:11.018Z 7 Acked 1006 Read 5 Done Sent Done false
5512024-01-10T17:55:11.018Z 8 Acked 1007 Write 5 Done Sent Done false
5522024-01-10T17:55:11.018Z 9 Acked 1008 Read 9 Done Sent Done false
5532024-01-10T17:55:11.018Z 10 Acked 1009 Read 10 Done Sent Done false
5542024-01-10T17:55:11.018Z 11 Acked 1010 Write 7 Done Sent Done false
5552024-01-10T17:55:11.018Z 12 Acked 1011 Write 7 Done Sent Done false
5562024-01-10T17:55:11.018Z 13 Acked 1012 Flush 0 Done Sent Done false
5572024-01-10T17:55:11.018Z 14 Acked 1013 Write 7 Done Sent Done false
5582024-01-10T17:55:11.018Z 15 Acked 1014 Write 9 Done Sent Done false
5592024-01-10T17:55:11.018Z 16 Acked 1015 Read 4 Done Sent Done false
5602024-01-10T17:55:11.018Z 17 Acked 1016 Read 3 Done Sent Done false
5612024-01-10T17:55:11.018Z 18 Acked 1017 Write 7 Done Sent Done false
5622024-01-10T17:55:11.019Z 19 Acked 1018 Read 7 Done Sent Done false
5632024-01-10T17:55:11.019Z 20 Acked 1019 Read 3 Done Sent Done false
5642024-01-10T17:55:11.019Z 21 Acked 1020 Read 6 Done Sent Done false
5652024-01-10T17:55:11.019Z 22 Acked 1021 Read 8 Done Sent Done false
5662024-01-10T17:55:11.019Z 23 Acked 1022 Write 7 Done Sent Done false
5672024-01-10T17:55:11.019Z 24 Acked 1023 Read 6 Done Sent Done false
5682024-01-10T17:55:11.019Z 25 Acked 1024 Write 4 Done Sent Done false
5692024-01-10T17:55:11.019Z 26 Acked 1025 Read 4 Done Sent Done false
5702024-01-10T17:55:11.019Z 27 Acked 1026 Write 9 Done Sent Done false
5712024-01-10T17:55:11.019Z 28 Acked 1027 Read 5 Done Sent Done false
5722024-01-10T17:55:11.019Z 29 Acked 1028 Write 3 Done Sent Done false
5732024-01-10T17:55:11.019Z 30 Acked 1029 Read 6 Sent Sent Done false
5742024-01-10T17:55:11.019Z STATES DS:0 DS:1 DS:2 TOTAL
5752024-01-10T17:55:11.019Z New 0 0 0 0
5762024-01-10T17:55:11.019Z Sent 1 28 0 29
5772024-01-10T17:55:11.019Z Done 29 2 30 61
5782024-01-10T17:55:11.019Z Skipped 0 0 0 0
5792024-01-10T17:55:11.019Z Error 0 0 0 0
5802024-01-10T17:55:11.019ZLast Flush: 1012 0 1012
5812024-01-10T17:55:11.019ZDownstairs last five completed:
5822024-01-10T17:55:11.020ZUpstairs last five completed: 30 29 28 27 26
5832024-01-10T17:55:11.020ZWrote out file "/tmp/test_repair_verify.data"
5842024-01-10T17:55:11.020Z
5852024-01-10T17:55:11.020ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5862024-01-10T17:55:11.020ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5872024-01-10T17:55:11.020Z 0 000-019 1 1 1 1 1 1 T F T <---
5882024-01-10T17:55:11.020Z 1 020-039 4 2 4 6 4 6 F T F <---
5892024-01-10T17:55:11.020Z 2 040-059 2 2 2 4 4 4 F F F
5902024-01-10T17:55:11.020Z 3 060-079 1 1 1 1 1 1 F F F
5912024-01-10T17:55:11.021Z 4 080-099 1 1 1 1 1 1 F F F
5922024-01-10T17:55:11.021Z 5 100-119 1 1 1 1 1 1 F F F
5932024-01-10T17:55:11.021Z 6 120-139 1 1 1 1 1 1 F F F
5942024-01-10T17:55:11.021Z 7 140-159 3 3 3 5 5 5 F F F
5952024-01-10T17:55:11.021Z 8 160-179 1 1 1 1 1 1 T F T <---
5962024-01-10T17:55:11.021Z 9 180-199 4 1 4 6 1 6 F F F <---
5972024-01-10T17:55:11.021Z 10 200-219 1 1 1 1 1 1 F F F
5982024-01-10T17:55:11.021Z 11 220-239 1 1 1 1 1 1 F F F
5992024-01-10T17:55:11.021Z 12 240-259 4 1 4 6 1 6 F F F <---
6002024-01-10T17:55:11.021Z 13 260-279 4 1 4 6 1 6 F F F <---
6012024-01-10T17:55:11.021Z 14 280-299 1 1 1 1 1 1 F F F
6022024-01-10T17:55:11.021Z 15 300-319 1 1 1 1 1 1 F F F
6032024-01-10T17:55:11.021Z 16 320-339 2 2 2 4 4 4 F F F
6042024-01-10T17:55:11.021Z 17 340-359 1 1 1 1 1 1 F F F
6052024-01-10T17:55:11.021Z 18 360-379 1 1 1 1 1 1 F F F
6062024-01-10T17:55:11.021Z 19 380-399 2 2 2 4 4 4 F F F
6072024-01-10T17:55:11.022Z 20 400-419 4 2 4 6 2 6 T F T <---
6082024-01-10T17:55:11.022Z 21 420-439 4 1 4 6 1 6 T F T <---
6092024-01-10T17:55:11.022Z 22 440-459 1 1 1 1 1 1 T F T <---
6102024-01-10T17:55:11.022Z 23 460-479 4 2 4 6 4 6 F F F <---
6112024-01-10T17:55:11.022Z 24 480-499 1 1 1 1 1 1 T F T <---
6122024-01-10T17:55:11.022Z 25 500-519 4 1 4 6 1 6 T F T <---
6132024-01-10T17:55:11.022Z 26 520-539 2 2 2 4 4 4 F F F
6142024-01-10T17:55:11.022Z 27 540-559 2 2 2 4 4 4 F F F
6152024-01-10T17:55:11.022Z 28 560-579 1 1 1 1 1 1 F F F
6162024-01-10T17:55:11.023Z 29 580-599 2 2 2 4 4 4 T F T <---
6172024-01-10T17:55:11.023ZMax gen: 4, Max flush: 6
6182024-01-10T17:55:11.023ZError: Difference in extent metadata found!
6192024-01-10T17:55:11.027ZA Difference in extent metadata is expected here
6202024-01-10T17:55:11.027ZOn loop 2
6212024-01-10T17:55:11.027Z
6222024-01-10T17:55:11.027ZVerifying data now
6232024-01-10T17:55:15.978ZNo change after verify
6242024-01-10T17:55:15.978ZLoop: 2 Downstairs dump after verify (and repair):
6252024-01-10T17:55:16.000ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6262024-01-10T17:55:16.000Z 0 000-019 5 5 5 7 7 7 F F F
6272024-01-10T17:55:16.000Z 1 020-039 4 4 4 6 6 6 F F F
6282024-01-10T17:55:16.000Z 2 040-059 2 2 2 4 4 4 F F F
6292024-01-10T17:55:16.000Z 3 060-079 1 1 1 1 1 1 F F F
6302024-01-10T17:55:16.001Z 4 080-099 1 1 1 1 1 1 F F F
6312024-01-10T17:55:16.001Z 5 100-119 1 1 1 1 1 1 F F F
6322024-01-10T17:55:16.002Z 6 120-139 1 1 1 1 1 1 F F F
6332024-01-10T17:55:16.002Z 7 140-159 3 3 3 5 5 5 F F F
6342024-01-10T17:55:16.002Z 8 160-179 5 5 5 7 7 7 F F F
6352024-01-10T17:55:16.002Z 9 180-199 4 4 4 6 6 6 F F F
6362024-01-10T17:55:16.002Z 10 200-219 1 1 1 1 1 1 F F F
6372024-01-10T17:55:16.005Z 11 220-239 1 1 1 1 1 1 F F F
6382024-01-10T17:55:16.005Z 12 240-259 4 4 4 6 6 6 F F F
6392024-01-10T17:55:16.005Z 13 260-279 4 4 4 6 6 6 F F F
6402024-01-10T17:55:16.005Z 14 280-299 1 1 1 1 1 1 F F F
6412024-01-10T17:55:16.005Z 15 300-319 1 1 1 1 1 1 F F F
6422024-01-10T17:55:16.006Z 16 320-339 2 2 2 4 4 4 F F F
6432024-01-10T17:55:16.006Z 17 340-359 1 1 1 1 1 1 F F F
6442024-01-10T17:55:16.006Z 18 360-379 1 1 1 1 1 1 F F F
6452024-01-10T17:55:16.006Z 19 380-399 2 2 2 4 4 4 F F F
6462024-01-10T17:55:16.006Z 20 400-419 5 5 5 7 7 7 F F F
6472024-01-10T17:55:16.006Z 21 420-439 5 5 5 7 7 7 F F F
6482024-01-10T17:55:16.006Z 22 440-459 5 5 5 7 7 7 F F F
6492024-01-10T17:55:16.006Z 23 460-479 4 4 4 6 6 6 F F F
6502024-01-10T17:55:16.006Z 24 480-499 5 5 5 7 7 7 F F F
6512024-01-10T17:55:16.006Z 25 500-519 5 5 5 7 7 7 F F F
6522024-01-10T17:55:16.006Z 26 520-539 2 2 2 4 4 4 F F F
6532024-01-10T17:55:16.006Z 27 540-559 2 2 2 4 4 4 F F F
6542024-01-10T17:55:16.006Z 28 560-579 1 1 1 1 1 1 F F F
6552024-01-10T17:55:16.007Z 29 580-599 5 5 5 7 7 7 F F F
6562024-01-10T17:55:16.007ZMax gen: 5, Max flush: 7
6572024-01-10T17:55:16.007Z
6582024-01-10T17:55:16.007ZBegin loop 3
6592024-01-10T17:55:16.007ZDownstairs to restart: 1
6602024-01-10T17:55:16.034Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.844934456Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179}
6612024-01-10T17:55:16.034Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.845454201Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179}
6622024-01-10T17:55:16.038Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.845517982Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179}
6632024-01-10T17:55:16.038Z{"msg":"Crucible f06c7c15-23b9-418f-b140-d7d334375eda has session id: 12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.845588221Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6642024-01-10T17:55:16.038Z{"msg":"Upstairs opts: Upstairs UUID: f06c7c15-23b9-418f-b140-d7d334375eda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.845685314Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6652024-01-10T17:55:16.039Z{"msg":"Crucible stats registered with UUID: f06c7c15-23b9-418f-b140-d7d334375eda","v":0,"name":"crucible","level":30Crucible runtime is spawned
6662024-01-10T17:55:16.039Z,"time":"2024-01-10T17:55:15.845785236Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6672024-01-10T17:55:16.039Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.845946324Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179}
6682024-01-10T17:55:16.039Z{"msg":"f06c7c15-23b9-418f-b140-d7d334375eda active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846083603Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6692024-01-10T17:55:16.040Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846162765Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"0","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6702024-01-10T17:55:16.040Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846188305Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"0","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6712024-01-10T17:55:16.040Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846211851Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"1","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6722024-01-10T17:55:16.040Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.84625144Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"1","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6732024-01-10T17:55:16.040Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846294452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"2","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6742024-01-10T17:55:16.040Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846337175Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"2","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6752024-01-10T17:55:16.041Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.84641229Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6762024-01-10T17:55:16.041Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846456339Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6772024-01-10T17:55:16.042Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846548393Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6782024-01-10T17:55:16.042Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846624332Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6792024-01-10T17:55:16.043Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846782038Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6802024-01-10T17:55:16.043Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.846896262Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6812024-01-10T17:55:16.043Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.848065278Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"2","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6822024-01-10T17:55:16.043Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.848177408Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"2","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6832024-01-10T17:55:16.043Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.848267303Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"0","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6842024-01-10T17:55:16.043Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.848313302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"0","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6852024-01-10T17:55:16.043Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.848541277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"1","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6862024-01-10T17:55:16.043Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.84860653Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"1","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6872024-01-10T17:55:16.043Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.849195445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"0","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6882024-01-10T17:55:16.043Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.849319649Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"2","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6892024-01-10T17:55:16.049Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.849694506Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"1","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6902024-01-10T17:55:16.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.850200581Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"0","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6912024-01-10T17:55:16.049Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.850292421Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"0","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6922024-01-10T17:55:16.049Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.850392812Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6932024-01-10T17:55:16.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.850434939Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"2","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6942024-01-10T17:55:16.049Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.850473916Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"2","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6952024-01-10T17:55:16.049Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.850531307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6962024-01-10T17:55:16.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.850762562Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"1","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6972024-01-10T17:55:16.049Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.850821995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"1","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6982024-01-10T17:55:16.049Z{"msg":"[0]R flush_numbers[0..12]: [7, 6, 4, 1, 1, 1, 1, 5, 7, 6, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.850867911Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
6992024-01-10T17:55:16.050Z{"msg":"[0]R generation[0..12]: [5, 4, 2, 1, 1, 1, 1, 3, 5, 4, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.850930759Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7002024-01-10T17:55:16.050Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
7012024-01-10T17:55:16.050Z,"time":"2024-01-10T17:55:15.850975365Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7022024-01-10T17:55:16.050Z{"msg":"[1]R flush_numbers[0..12]: [7, 6, 4, 1, 1, 1, 1, 5, 7, 6, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.85103764Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7032024-01-10T17:55:16.050Z{"msg":"[1]R generation[0..12]: [5, 4, 2, 1, 1, 1, 1, 3, 5, 4, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851098725Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7042024-01-10T17:55:16.050Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851145018Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7052024-01-10T17:55:16.050Z{"msg":"[2]R flush_numbers[0..12]: [7, 6, 4, 1, 1, 1, 1, 5, 7, 6, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851198178Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7062024-01-10T17:55:16.050Z{"msg":"[2]R generation[0..12]: [5, 4, 2, 1, 1, 1, 1, 3, 5, 4, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851241084Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7072024-01-10T17:55:16.050Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851303894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7082024-01-10T17:55:16.050Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851347389Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7092024-01-10T17:55:16.050Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851389765Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7102024-01-10T17:55:16.050Z{"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851444325Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7112024-01-10T17:55:16.051Z{"msg":"All extents match","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7122024-01-10T17:55:16.051Z,"time":"2024-01-10T17:55:15.851485721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"":"downstairs","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7132024-01-10T17:55:16.051Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851603822Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7142024-01-10T17:55:16.051Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851669686Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"0","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7152024-01-10T17:55:16.051Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851710292Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"1","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7162024-01-10T17:55:16.051Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851767095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"client":"2","":"downstairs","session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7172024-01-10T17:55:16.051Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851810071Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7182024-01-10T17:55:16.051Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851867696Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7192024-01-10T17:55:16.051Z{"msg":"f06c7c15-23b9-418f-b140-d7d334375eda is now active with session: 12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851910845Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7202024-01-10T17:55:16.052Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.851966522Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179,"session_id":"12ce72be-07fa-48a6-bfe2-3e7bf4e76a7f"}
7212024-01-10T17:55:16.052Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:15.852022358Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1179}
7222024-01-10T17:55:16.052ZLoading write count information from file "/tmp/test_repair_verify.data"
7232024-01-10T17:55:16.052ZRun Repair workload
7242024-01-10T17:55:16.052Z01/30 Write block 490 len 40960 data: 2 2 2 2 2 3 3 3 3 3
7252024-01-10T17:55:16.053Z02/30 Write block 324 len 8192 data: 2 2
7262024-01-10T17:55:16.053Z03/30 Read block 237 len 28672
7272024-01-10T17:55:16.053Z04/30 Read block 273 len 32768
7282024-01-10T17:55:16.056Z05/30 Write block 341 len 16384 data: 2 2 2 2
7292024-01-10T17:55:16.056Z06/30 Write block 140 len 28672 data: 2 2 2 2 2 2 2
7302024-01-10T17:55:16.056Z07/30 Write block 162 len 28672 data: 3 3 3 3 2 2 2
7312024-01-10T17:55:16.056Z08/30 Read block 235 len 12288
7322024-01-10T17:55:16.056Z09/30 Flush
7332024-01-10T17:55:16.059Z10/30 Write block 46 len 40960 data: 2 2 2 2 2 2 2 2 3 3
7342024-01-10T17:55:16.062Z11/30 Read block 436 len 4096
7352024-01-10T17:55:16.062Z12/30 Write block 536 len 32768 data: 2 2 2 2 2 2 2 2
7362024-01-10T17:55:16.065Z13/30 Read block 501 len 12288
7372024-01-10T17:55:16.065Z14/30 Write block 141 len 16384 data: 3 3 3 3
7382024-01-10T17:55:16.068Z15/30 Read block 355 len 28672
7392024-01-10T17:55:16.071Z16/30 Read block 546 len 40960
7402024-01-10T17:55:16.074Z17/30 Read block 384 len 24576
7412024-01-10T17:55:16.077Z18/30 Write block 216 len 28672 data: 2 2 2 2 2 2 2
7422024-01-10T17:55:16.078Z19/30 Flush
7432024-01-10T17:55:16.081Z20/30 Write block 304 len 24576 data: 2 2 2 2 2 2
7442024-01-10T17:55:16.082Z21/30 Read block 304 len 40960
7452024-01-10T17:55:16.084Z22/30 Write block 79 len 12288 data: 2 2 2
7462024-01-10T17:55:16.087Z23/30 Write block 439 len 32768 data: 2 2 2 2 2 2 3 3
7472024-01-10T17:55:16.087Z24/30 Read block 71 len 28672
7482024-01-10T17:55:16.092Z25/30 Read block 111 len 40960
7492024-01-10T17:55:16.096Z26/30 Write block 591 len 36864 data: 2 2 2 2 2 2 2 2 2
7502024-01-10T17:55:16.097Z27/30 Write block 199 len 20480 data: 2 2 2 2 2
7512024-01-10T17:55:16.097Z28/30 Read block 38 len 20480
7522024-01-10T17:55:16.107Z29/30 Read block 258 len 28672
7532024-01-10T17:55:16.107Z30/30 Read block 412 len 32768
7542024-01-10T17:55:16.116Z----------------------------------------------------------------
7552024-01-10T17:55:16.116Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7562024-01-10T17:55:16.116ZStates: Active Active Active
7572024-01-10T17:55:16.116ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7582024-01-10T17:55:16.116Z 1 Acked 1000 Write 10 Done Done Done false
7592024-01-10T17:55:16.116Z 2 Acked 1001 Write 2 Done Done Done false
7602024-01-10T17:55:16.116Z 3 Acked 1002 Read 7 Done Done Done false
7612024-01-10T17:55:16.116Z 4 Acked 1003 Read 8 Done Sent Done false
7622024-01-10T17:55:16.116Z 5 Acked 1004 Write 4 Done Sent Done false
7632024-01-10T17:55:16.116Z 6 Acked 1005 Write 7 Done Sent Done false
7642024-01-10T17:55:16.116Z 7 Acked 1006 Write 7 Done Sent Done false
7652024-01-10T17:55:16.117Z 8 Acked 1007 Read 3 Done Sent Done false
7662024-01-10T17:55:16.117Z 9 Acked 1008 Flush 0 Done Sent Done false
7672024-01-10T17:55:16.117Z 10 Acked 1009 Write 10 Done Sent Done false
7682024-01-10T17:55:16.119Z 11 Acked 1010 Read 1 Done Sent Done false
7692024-01-10T17:55:16.119Z 12 Acked 1011 Write 8 Done Sent Done false
7702024-01-10T17:55:16.119Z 13 Acked 1012 Read 3 Done Sent Done false
7712024-01-10T17:55:16.119Z 14 Acked 1013 Write 4 Done Sent Done false
7722024-01-10T17:55:16.119Z 15 Acked 1014 Read 7 Done Sent Done false
7732024-01-10T17:55:16.119Z 16 Acked 1015 Read 10 Done Sent Done false
7742024-01-10T17:55:16.119Z 17 Acked 1016 Read 6 Done Sent Done false
7752024-01-10T17:55:16.119Z 18 Acked 1017 Write 7 Done Sent Done false
7762024-01-10T17:55:16.119Z 19 Acked 1018 Flush 0 Done Sent Done false
7772024-01-10T17:55:16.119Z 20 Acked 1019 Write 6 Done Sent Done false
7782024-01-10T17:55:16.119Z 21 Acked 1020 Read 10 Done Sent Done false
7792024-01-10T17:55:16.119Z 22 Acked 1021 Write 3 Done Sent Done false
7802024-01-10T17:55:16.119Z 23 Acked 1022 Write 8 Done Sent Done false
7812024-01-10T17:55:16.119Z 24 Acked 1023 Read 7 Done Sent Done false
7822024-01-10T17:55:16.119Z 25 Acked 1024 Read 10 Done Sent Done false
7832024-01-10T17:55:16.119Z 26 Acked 1025 Write 9 Done Sent Done false
7842024-01-10T17:55:16.119Z 27 Acked 1026 Write 5 Done Sent Done false
7852024-01-10T17:55:16.119Z 28 Acked 1027 Read 5 Done Sent Done false
7862024-01-10T17:55:16.119Z 29 Acked 1028 Read 7 Done Sent Done false
7872024-01-10T17:55:16.119Z 30 Acked 1029 Read 8 Done Sent Sent false
7882024-01-10T17:55:16.119Z STATES DS:0 DS:1 DS:2 TOTAL
7892024-01-10T17:55:16.120Z New 0 0 0 0
7902024-01-10T17:55:16.120Z Sent 0 27 1 28
7912024-01-10T17:55:16.120Z Done 30 3 29 62
7922024-01-10T17:55:16.120Z Skipped 0 0 0 0
7932024-01-10T17:55:16.120Z Error 0 0 0 0
7942024-01-10T17:55:16.120ZLast Flush: 1018 0 1018
7952024-01-10T17:55:16.120ZDownstairs last five completed:
7962024-01-10T17:55:16.120ZUpstairs last five completed: 30 29 28 27 26
7972024-01-10T17:55:16.120ZWrote out file "/tmp/test_repair_verify.data"
7982024-01-10T17:55:16.120Z
7992024-01-10T17:55:16.129ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8002024-01-10T17:55:16.143ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8012024-01-10T17:55:16.143Z 0 000-019 5 5 5 7 7 7 F F F
8022024-01-10T17:55:16.143Z 1 020-039 4 4 4 6 6 6 F F F
8032024-01-10T17:55:16.143Z 2 040-059 6 2 6 9 4 9 F F F <---
8042024-01-10T17:55:16.143Z 3 060-079 1 1 1 1 1 1 T F T <---
8052024-01-10T17:55:16.143Z 4 080-099 1 1 1 1 1 1 T F T <---
8062024-01-10T17:55:16.150Z 5 100-119 1 1 1 1 1 1 F F F
8072024-01-10T17:55:16.150Z 6 120-139 1 1 1 1 1 1 F F F
8082024-01-10T17:55:16.150Z 7 140-159 6 3 6 9 5 9 F F F <---
8092024-01-10T17:55:16.150Z 8 160-179 6 5 6 8 7 8 F F F <---
8102024-01-10T17:55:16.150Z 9 180-199 4 4 4 6 6 6 T F T <---
8112024-01-10T17:55:16.150Z 10 200-219 6 1 6 9 1 9 T F T <---
8122024-01-10T17:55:16.150Z 11 220-239 6 1 6 9 1 9 F F F <---
8132024-01-10T17:55:16.150Z 12 240-259 4 4 4 6 6 6 F F F
8142024-01-10T17:55:16.151Z 13 260-279 4 4 4 6 6 6 F F F
8152024-01-10T17:55:16.151Z 14 280-299 1 1 1 1 1 1 F F F
8162024-01-10T17:55:16.151Z 15 300-319 1 1 1 1 1 1 T F T <---
8172024-01-10T17:55:16.151Z 16 320-339 6 2 6 8 4 8 F T F <---
8182024-01-10T17:55:16.151Z 17 340-359 6 1 6 8 1 8 F F F <---
8192024-01-10T17:55:16.151Z 18 360-379 1 1 1 1 1 1 F F F
8202024-01-10T17:55:16.151Z 19 380-399 2 2 2 4 4 4 F F F
8212024-01-10T17:55:16.151Z 20 400-419 5 5 5 7 7 7 F F F
8222024-01-10T17:55:16.151Z 21 420-439 5 5 5 7 7 7 T F T <---
8232024-01-10T17:55:16.152Z 22 440-459 5 5 5 7 7 7 T F T <---
8242024-01-10T17:55:16.152Z 23 460-479 4 4 4 6 6 6 F F F
8252024-01-10T17:55:16.152Z 24 480-499 6 5 6 8 7 8 F T F <---
8262024-01-10T17:55:16.152Z 25 500-519 5 5 5 7 7 7 F F F
8272024-01-10T17:55:16.152Z 26 520-539 6 2 6 9 4 9 F F F <---
8282024-01-10T17:55:16.152Z 27 540-559 6 2 6 9 4 9 F F F <---
8292024-01-10T17:55:16.152Z 28 560-579 1 1 1 1 1 1 F F F
8302024-01-10T17:55:16.152Z 29 580-599 5 5 5 7 7 7 T F T <---
8312024-01-10T17:55:16.152ZMax gen: 6, Max flush: 9
8322024-01-10T17:55:16.152ZError: Difference in extent metadata found!
8332024-01-10T17:55:16.152ZA Difference in extent metadata is expected here
8342024-01-10T17:55:16.152ZOn loop 3
8352024-01-10T17:55:16.152Z
8362024-01-10T17:55:16.158ZVerifying data now
8372024-01-10T17:55:22.750ZNo change after verify
8382024-01-10T17:55:22.750ZLoop: 3 Downstairs dump after verify (and repair):
8392024-01-10T17:55:22.772ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8402024-01-10T17:55:22.772Z 0 000-019 5 5 5 7 7 7 F F F
8412024-01-10T17:55:22.772Z 1 020-039 4 4 4 6 6 6 F F F
8422024-01-10T17:55:22.772Z 2 040-059 6 6 6 9 9 9 F F F
8432024-01-10T17:55:22.772Z 3 060-079 7 7 7 10 10 10 F F F
8442024-01-10T17:55:22.772Z 4 080-099 7 7 7 10 10 10 F F F
8452024-01-10T17:55:22.772Z 5 100-119 1 1 1 1 1 1 F F F
8462024-01-10T17:55:22.772Z 6 120-139 1 1 1 1 1 1 F F F
8472024-01-10T17:55:22.772Z 7 140-159 6 6 6 9 9 9 F F F
8482024-01-10T17:55:22.773Z 8 160-179 6 6 6 8 8 8 F F F
8492024-01-10T17:55:22.776Z 9 180-199 7 7 7 10 10 10 F F F
8502024-01-10T17:55:22.776Z 10 200-219 7 7 7 10 10 10 F F F
8512024-01-10T17:55:22.776Z 11 220-239 6 6 6 9 9 9 F F F
8522024-01-10T17:55:22.776Z 12 240-259 4 4 4 6 6 6 F F F
8532024-01-10T17:55:22.776Z 13 260-279 4 4 4 6 6 6 F F F
8542024-01-10T17:55:22.776Z 14 280-299 1 1 1 1 1 1 F F F
8552024-01-10T17:55:22.776Z 15 300-319 7 7 7 10 10 10 F F F
8562024-01-10T17:55:22.776Z 16 320-339 6 6 6 8 8 8 F F F
8572024-01-10T17:55:22.777Z 17 340-359 6 6 6 8 8 8 F F F
8582024-01-10T17:55:22.777Z 18 360-379 1 1 1 1 1 1 F F F
8592024-01-10T17:55:22.777Z 19 380-399 2 2 2 4 4 4 F F F
8602024-01-10T17:55:22.777Z 20 400-419 5 5 5 7 7 7 F F F
8612024-01-10T17:55:22.780Z 21 420-439 7 7 7 10 10 10 F F F
8622024-01-10T17:55:22.780Z 22 440-459 7 7 7 10 10 10 F F F
8632024-01-10T17:55:22.780Z 23 460-479 4 4 4 6 6 6 F F F
8642024-01-10T17:55:22.781Z 24 480-499 6 6 6 8 8 8 F F F
8652024-01-10T17:55:22.781Z 25 500-519 5 5 5 7 7 7 F F F
8662024-01-10T17:55:22.781Z 26 520-539 6 6 6 9 9 9 F F F
8672024-01-10T17:55:22.781Z 27 540-559 6 6 6 9 9 9 F F F
8682024-01-10T17:55:22.781Z 28 560-579 1 1 1 1 1 1 F F F
8692024-01-10T17:55:22.781Z 29 580-599 7 7 7 10 10 10 F F F
8702024-01-10T17:55:22.781ZMax gen: 7, Max flush: 10
8712024-01-10T17:55:22.784Z
8722024-01-10T17:55:22.784ZBegin loop 4
8732024-01-10T17:55:22.784ZDownstairs to restart: 0
8742024-01-10T17:55:22.802Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.614323261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187}
8752024-01-10T17:55:22.806Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.614829767Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187}
8762024-01-10T17:55:22.806Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.614880223Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187}
8772024-01-10T17:55:22.806Z{"msg":"Crucible d2c151bf-136d-40d7-bc08-ae5cc0cae799 has session id: 52e77e42-fd7c-4146-82bb-3a1bf3e1341d","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.614920199Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8782024-01-10T17:55:22.806Z{"msg":"Crucible runtime is spawned
8792024-01-10T17:55:22.806ZUpstairs opts: Upstairs UUID: d2c151bf-136d-40d7-bc08-ae5cc0cae799, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.61506284Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8802024-01-10T17:55:22.806Z{"msg":"Crucible stats registered with UUID: d2c151bf-136d-40d7-bc08-ae5cc0cae799","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615149804Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8812024-01-10T17:55:22.806Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615193062Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187}
8822024-01-10T17:55:22.806Z{"msg":"d2c151bf-136d-40d7-bc08-ae5cc0cae799 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615396367Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8832024-01-10T17:55:22.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615471119Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"0","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8842024-01-10T17:55:22.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615544255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"0","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8852024-01-10T17:55:22.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615591815Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"1","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8862024-01-10T17:55:22.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615635058Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"1","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8872024-01-10T17:55:22.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615681533Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"2","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8882024-01-10T17:55:22.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615725006Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"2","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8892024-01-10T17:55:22.807Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615798417Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8902024-01-10T17:55:22.807Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615843062Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8912024-01-10T17:55:22.807Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615894285Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8922024-01-10T17:55:22.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.615939055Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"io task","client":"1","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8932024-01-10T17:55:22.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.616096154Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"io task","client":"2","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8942024-01-10T17:55:22.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.616179779Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"io task","client":"0","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8952024-01-10T17:55:22.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.617335117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"1","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8962024-01-10T17:55:22.807Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.617394699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"1","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8972024-01-10T17:55:22.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.617419613Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"2","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8982024-01-10T17:55:22.811Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.617442188Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"2","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
8992024-01-10T17:55:22.815Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.618418675Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"2","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9002024-01-10T17:55:22.815Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.618567306Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"0","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9012024-01-10T17:55:22.815Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.618715429Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"0","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9022024-01-10T17:55:22.815Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.618797022Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"1","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9032024-01-10T17:55:22.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.619520426Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"2","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9042024-01-10T17:55:22.815Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.619593591Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"2","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9052024-01-10T17:55:22.815Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.6196641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9062024-01-10T17:55:22.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.619728832Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"1","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9072024-01-10T17:55:22.815Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.619756222Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"1","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9082024-01-10T17:55:22.815Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.619811897Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9092024-01-10T17:55:22.815Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.619850271Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"0","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9102024-01-10T17:55:22.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.620443697Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"0","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9112024-01-10T17:55:22.815Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.620525351Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"0","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9122024-01-10T17:55:22.816Z{"msg":"[0]R flush_numbers[0..12]: [7, 6, 9, 10, 10, 1, 1, 9, 8, 10, 10, 9]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.620597719Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9132024-01-10T17:55:22.816Z{"msg":"[0]R generation[0..12]: [5, 4, 6, 7, 7, 1, 1, 6, 6, 7, 7, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.620668076Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9142024-01-10T17:55:22.816Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.620740859Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9152024-01-10T17:55:22.816Z{"msg":"[1]R flush_numbers[0..12]: [7, 6, 9, 10, 10, 1, 1, 9, 8, 10, 10, 9]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.620786243Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9162024-01-10T17:55:22.816ZWait for a query_work_queue command to finish before sending IO
9172024-01-10T17:55:22.816Z{"msg":"[1]R generation[0..12]: [5, 4, 6, 7, 7, 1, 1, 6, 6, 7, 7, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.620864503Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9182024-01-10T17:55:22.816Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.620928442Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9192024-01-10T17:55:22.816Z{"msg":"[2]R flush_numbers[0..12]: [7, 6, 9, 10, 10, 1, 1, 9, 8, 10, 10, 9]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.620954843Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9202024-01-10T17:55:22.816Z{"msg":"[2]R generation[0..12]: [5, 4, 6, 7, 7, 1, 1, 6, 6, 7, 7, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621012137Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9212024-01-10T17:55:22.816Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621043014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9222024-01-10T17:55:22.816Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621065066Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9232024-01-10T17:55:22.816Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621088208Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9242024-01-10T17:55:22.816Z{"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621128954Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9252024-01-10T17:55:22.816Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621151727Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"":"downstairs","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9262024-01-10T17:55:22.816Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621205527Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9272024-01-10T17:55:22.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621226863Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"0","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9282024-01-10T17:55:22.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621248825Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"1","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9292024-01-10T17:55:22.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621288631Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"client":"2","":"downstairs","session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9302024-01-10T17:55:22.817Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621311698Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9312024-01-10T17:55:22.817Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621331835Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9322024-01-10T17:55:22.817Z{"msg":"d2c151bf-136d-40d7-bc08-ae5cc0cae799 is now active with session: 52e77e42-fd7c-4146-82bb-3a1bf3e1341d","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621383551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9332024-01-10T17:55:22.817Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621404112Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187,"session_id":"52e77e42-fd7c-4146-82bb-3a1bf3e1341d"}
9342024-01-10T17:55:22.817Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:22.621441772Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1187}
9352024-01-10T17:55:22.817ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9362024-01-10T17:55:22.817ZLoading write count information from file "/tmp/test_repair_verify.data"
9372024-01-10T17:55:22.817ZRun Repair workload
9382024-01-10T17:55:22.817Z01/30 Write block 335 len 36864 data: 2 2 2 2 2 2 3 3 3
9392024-01-10T17:55:22.817Z02/30 Read block 139 len 20480
9402024-01-10T17:55:22.821Z03/30 Read block 548 len 12288
9412024-01-10T17:55:22.821Z04/30 Read block 462 len 36864
9422024-01-10T17:55:22.821Z05/30 Read block 308 len 8192
9432024-01-10T17:55:22.824Z06/30 Write block 444 len 12288 data: 3 4 4
9442024-01-10T17:55:22.824Z07/30 Write block 564 len 8192 data: 2 2
9452024-01-10T17:55:22.824Z08/30 Flush
9462024-01-10T17:55:22.830Z09/30 Write block 258 len 4096 data: 2
9472024-01-10T17:55:22.830Z10/30 Write block 499 len 36864 data: 4 3 3 3 3 2 2 2 3
9482024-01-10T17:55:22.833Z11/30 Read block 416 len 40960
9492024-01-10T17:55:22.833Z12/30 Write block 516 len 28672 data: 3 3 2 2 2 3 2
9502024-01-10T17:55:22.836Z13/30 Read block 446 len 12288
9512024-01-10T17:55:22.840Z14/30 Write block 12 len 40960 data: 3 3 3 3 3 3 3 2 2 3
9522024-01-10T17:55:22.840Z15/30 Read block 418 len 20480
9532024-01-10T17:55:22.843Z16/30 Read block 514 len 40960
9542024-01-10T17:55:22.847Z17/30 Write block 358 len 4096 data: 2
9552024-01-10T17:55:22.847Z18/30 Write block 345 len 24576 data: 2 2 2 2 2 2
9562024-01-10T17:55:22.847Z19/30 Read block 241 len 28672
9572024-01-10T17:55:22.850Z20/30 Write block 324 len 28672 data: 3 3 2 2 2 2 2
9582024-01-10T17:55:22.853Z21/30 Write block 208 len 40960 data: 2 2 2 2 2 2 2 2 3 3
9592024-01-10T17:55:22.853Z22/30 Read block 494 len 4096
9602024-01-10T17:55:22.855Z23/30 Read block 125 len 4096
9612024-01-10T17:55:22.855Z24/30 Flush
9622024-01-10T17:55:22.859Z25/30 Flush
9632024-01-10T17:55:22.863Z26/30 Write block 108 len 36864 data: 2 2 2 2 2 2 2 2 2
9642024-01-10T17:55:22.863Z27/30 Read block 315 len 12288
9652024-01-10T17:55:22.863Z28/30 Read block 419 len 32768
9662024-01-10T17:55:22.866Z29/30 Write block 296 len 24576 data: 2 2 2 2 2 2
9672024-01-10T17:55:22.870Z30/30 Write block 544 len 16384 data: 3 3 3 3
9682024-01-10T17:55:22.870Z----------------------------------------------------------------
9692024-01-10T17:55:22.871Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9702024-01-10T17:55:22.871ZStates: Active Active Active
9712024-01-10T17:55:22.871ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9722024-01-10T17:55:22.871Z 1 Acked 1000 Write 9 Done Done Done false
9732024-01-10T17:55:22.871Z 2 Acked 1001 Read 5 Done Done Done false
9742024-01-10T17:55:22.871Z 3 Acked 1002 Read 3 Done Done Done false
9752024-01-10T17:55:22.871Z 4 Acked 1003 Read 9 Done Done Done false
9762024-01-10T17:55:22.871Z 5 Acked 1004 Read 2 Done Done Done false
9772024-01-10T17:55:22.871Z 6 Acked 1005 Write 3 Sent Done Done false
9782024-01-10T17:55:22.871Z 7 Acked 1006 Write 2 Sent Done Done false
9792024-01-10T17:55:22.871Z 8 Acked 1007 Flush 0 Sent Done Done false
9802024-01-10T17:55:22.871Z 9 Acked 1008 Write 1 Sent Done Done false
9812024-01-10T17:55:22.871Z 10 Acked 1009 Write 9 Sent Done Done false
9822024-01-10T17:55:22.871Z 11 Acked 1010 Read 10 Sent Done Done false
9832024-01-10T17:55:22.871Z 12 Acked 1011 Write 7 Sent Done Done false
9842024-01-10T17:55:22.871Z 13 Acked 1012 Read 3 Sent Done Done false
9852024-01-10T17:55:22.871Z 14 Acked 1013 Write 10 Sent Done Done false
9862024-01-10T17:55:22.872Z 15 Acked 1014 Read 5 Sent Done Done false
9872024-01-10T17:55:22.872Z 16 Acked 1015 Read 10 Sent Done Done false
9882024-01-10T17:55:22.872Z 17 Acked 1016 Write 1 Sent Done Done false
9892024-01-10T17:55:22.872Z 18 Acked 1017 Write 6 Sent Done Done false
9902024-01-10T17:55:22.872Z 19 Acked 1018 Read 7 Sent Done Done false
9912024-01-10T17:55:22.872Z 20 Acked 1019 Write 7 Sent Done Done false
9922024-01-10T17:55:22.873Z 21 Acked 1020 Write 10 Sent Done Done false
9932024-01-10T17:55:22.873Z 22 Acked 1021 Read 1 Sent Done Done false
9942024-01-10T17:55:22.873Z 23 Acked 1022 Read 1 Sent Done Done false
9952024-01-10T17:55:22.873Z 24 Acked 1023 Flush 0 Sent Done Done false
9962024-01-10T17:55:22.873Z 25 Acked 1024 Flush 0 Sent Done Done false
9972024-01-10T17:55:22.874Z 26 Acked 1025 Write 9 Sent Done Done false
9982024-01-10T17:55:22.874Z 27 Acked 1026 Read 3 Sent Done Done false
9992024-01-10T17:55:22.874Z 28 Acked 1027 Read 8 Sent Done Done false
10002024-01-10T17:55:22.874Z 29 Acked 1028 Write 6 Sent Sent Sent false
10012024-01-10T17:55:22.874Z 30 Acked 1029 Write 4 Sent Sent Sent false
10022024-01-10T17:55:22.874Z STATES DS:0 DS:1 DS:2 TOTAL
10032024-01-10T17:55:22.874Z New 0 0 0 0
10042024-01-10T17:55:22.874Z Sent 25 2 2 29
10052024-01-10T17:55:22.874Z Done 5 28 28 61
10062024-01-10T17:55:22.874Z Skipped 0 0 0 0
10072024-01-10T17:55:22.874Z Error 0 0 0 0
10082024-01-10T17:55:22.874ZLast Flush: 0 1024 1024
10092024-01-10T17:55:22.874ZDownstairs last five completed:
10102024-01-10T17:55:22.874ZUpstairs last five completed: 30 29 28 27 26
10112024-01-10T17:55:22.874ZWrote out file "/tmp/test_repair_verify.data"
10122024-01-10T17:55:22.877Z
10132024-01-10T17:55:22.881ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10142024-01-10T17:55:22.906ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10152024-01-10T17:55:22.906Z 0 000-019 5 8 8 7 12 12 F F F <---
10162024-01-10T17:55:22.915Z 1 020-039 4 8 8 6 12 12 F F F <---
10172024-01-10T17:55:22.915Z 2 040-059 6 6 6 9 9 9 F F F
10182024-01-10T17:55:22.915Z 3 060-079 7 7 7 10 10 10 F F F
10192024-01-10T17:55:22.918Z 4 080-099 7 7 7 10 10 10 F F F
10202024-01-10T17:55:22.918Z 5 100-119 1 1 1 1 1 1 F T T <---
10212024-01-10T17:55:22.918Z 6 120-139 1 1 1 1 1 1 F F F
10222024-01-10T17:55:22.919Z 7 140-159 6 6 6 9 9 9 F F F
10232024-01-10T17:55:22.919Z 8 160-179 6 6 6 8 8 8 F F F
10242024-01-10T17:55:22.919Z 9 180-199 7 7 7 10 10 10 F F F
10252024-01-10T17:55:22.919Z 10 200-219 7 8 8 10 12 12 F F F <---
10262024-01-10T17:55:22.919Z 11 220-239 6 6 6 9 9 9 F F F
10272024-01-10T17:55:22.919Z 12 240-259 4 8 8 6 12 12 F F F <---
10282024-01-10T17:55:22.919Z 13 260-279 4 4 4 6 6 6 F F F
10292024-01-10T17:55:22.919Z 14 280-299 1 1 1 1 1 1 F T T <---
10302024-01-10T17:55:22.919Z 15 300-319 7 7 7 10 10 10 F T T <---
10312024-01-10T17:55:22.919Z 16 320-339 6 8 8 8 12 12 T F F <---
10322024-01-10T17:55:22.919Z 17 340-359 6 8 8 8 12 12 T F F <---
10332024-01-10T17:55:22.919Z 18 360-379 1 1 1 1 1 1 F F F
10342024-01-10T17:55:22.919Z 19 380-399 2 2 2 4 4 4 F F F
10352024-01-10T17:55:22.919Z 20 400-419 5 5 5 7 7 7 F F F
10362024-01-10T17:55:22.919Z 21 420-439 7 7 7 10 10 10 F F F
10372024-01-10T17:55:22.919Z 22 440-459 7 8 8 10 11 11 F F F <---
10382024-01-10T17:55:22.919Z 23 460-479 4 4 4 6 6 6 F F F
10392024-01-10T17:55:22.919Z 24 480-499 6 8 8 8 12 12 F F F <---
10402024-01-10T17:55:22.919Z 25 500-519 5 8 8 7 12 12 F F F <---
10412024-01-10T17:55:22.920Z 26 520-539 6 8 8 9 12 12 F F F <---
10422024-01-10T17:55:22.920Z 27 540-559 6 6 6 9 9 9 F T T <---
10432024-01-10T17:55:22.920Z 28 560-579 1 8 8 1 11 11 F F F <---
10442024-01-10T17:55:22.920Z 29 580-599 7 7 7 10 10 10 F F F
10452024-01-10T17:55:22.920ZMax gen: 8, Max flush: 12
10462024-01-10T17:55:22.920ZError: Difference in extent metadata found!
10472024-01-10T17:55:22.920ZA Difference in extent metadata is expected here
10482024-01-10T17:55:22.920ZOn loop 4
10492024-01-10T17:55:22.920Z
10502024-01-10T17:55:22.920ZVerifying data now
10512024-01-10T17:55:29.336ZNo change after verify
10522024-01-10T17:55:29.336ZLoop: 4 Downstairs dump after verify (and repair):
10532024-01-10T17:55:29.359ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10542024-01-10T17:55:29.359Z 0 000-019 8 8 8 12 12 12 F F F
10552024-01-10T17:55:29.359Z 1 020-039 8 8 8 12 12 12 F F F
10562024-01-10T17:55:29.359Z 2 040-059 6 6 6 9 9 9 F F F
10572024-01-10T17:55:29.359Z 3 060-079 7 7 7 10 10 10 F F F
10582024-01-10T17:55:29.359Z 4 080-099 7 7 7 10 10 10 F F F
10592024-01-10T17:55:29.359Z 5 100-119 9 9 9 13 13 13 F F F
10602024-01-10T17:55:29.359Z 6 120-139 1 1 1 1 1 1 F F F
10612024-01-10T17:55:29.359Z 7 140-159 6 6 6 9 9 9 F F F
10622024-01-10T17:55:29.359Z 8 160-179 6 6 6 8 8 8 F F F
10632024-01-10T17:55:29.362Z 9 180-199 7 7 7 10 10 10 F F F
10642024-01-10T17:55:29.362Z 10 200-219 8 8 8 12 12 12 F F F
10652024-01-10T17:55:29.362Z 11 220-239 6 6 6 9 9 9 F F F
10662024-01-10T17:55:29.362Z 12 240-259 8 8 8 12 12 12 F F F
10672024-01-10T17:55:29.362Z 13 260-279 4 4 4 6 6 6 F F F
10682024-01-10T17:55:29.362Z 14 280-299 9 9 9 13 13 13 F F F
10692024-01-10T17:55:29.363Z 15 300-319 9 9 9 13 13 13 F F F
10702024-01-10T17:55:29.363Z 16 320-339 8 8 8 12 12 12 F F F
10712024-01-10T17:55:29.363Z 17 340-359 8 8 8 12 12 12 F F F
10722024-01-10T17:55:29.363Z 18 360-379 1 1 1 1 1 1 F F F
10732024-01-10T17:55:29.363Z 19 380-399 2 2 2 4 4 4 F F F
10742024-01-10T17:55:29.363Z 20 400-419 5 5 5 7 7 7 F F F
10752024-01-10T17:55:29.363Z 21 420-439 7 7 7 10 10 10 F F F
10762024-01-10T17:55:29.363Z 22 440-459 8 8 8 11 11 11 F F F
10772024-01-10T17:55:29.363Z 23 460-479 4 4 4 6 6 6 F F F
10782024-01-10T17:55:29.363Z 24 480-499 8 8 8 12 12 12 F F F
10792024-01-10T17:55:29.363Z 25 500-519 8 8 8 12 12 12 F F F
10802024-01-10T17:55:29.363Z 26 520-539 8 8 8 12 12 12 F F F
10812024-01-10T17:55:29.363Z 27 540-559 9 9 9 13 13 13 F F F
10822024-01-10T17:55:29.363Z 28 560-579 8 8 8 11 11 11 F F F
10832024-01-10T17:55:29.363Z 29 580-599 7 7 7 10 10 10 F F F
10842024-01-10T17:55:29.363ZMax gen: 9, Max flush: 13
10852024-01-10T17:55:29.366Z
10862024-01-10T17:55:29.366ZBegin loop 5
10872024-01-10T17:55:29.366ZDownstairs to restart: 2
10882024-01-10T17:55:29.430Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.206784146Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195}
10892024-01-10T17:55:29.430Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.207381587Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195}
10902024-01-10T17:55:29.430Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.207418401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195}
10912024-01-10T17:55:29.435Z{"msg":"Crucible bf811484-668a-4729-afe8-511778dbdce6 has session id: 40a26109-29dd-4683-ab43-f181ce855cd3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.207485095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
10922024-01-10T17:55:29.435Z{"msg":"Upstairs opts: Upstairs UUID: bf811484-668a-4729-afe8-511778dbdce6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.207638175Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
10932024-01-10T17:55:29.435Z{"msg":"Crucible stats registered with UUID: bf811484-668a-4729-afe8-511778dbdce6","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.207669711Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
10942024-01-10T17:55:29.435ZCrucible runtime is spawned
10952024-01-10T17:55:29.436Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.207929602Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195}
10962024-01-10T17:55:29.436Z{"msg":"bf811484-668a-4729-afe8-511778dbdce6 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.207994621Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
10972024-01-10T17:55:29.436Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.208178189Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"0","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
10982024-01-10T17:55:29.436Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.20827147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"0","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
10992024-01-10T17:55:29.436Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.208410997Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"1","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11002024-01-10T17:55:29.436Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.208565088Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"1","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11012024-01-10T17:55:29.436Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.208677535Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"2","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11022024-01-10T17:55:29.438Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.208770036Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"2","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11032024-01-10T17:55:29.438Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.208817971Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11042024-01-10T17:55:29.438Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.208890746Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11052024-01-10T17:55:29.438Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.209046908Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11062024-01-10T17:55:29.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.209118286Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11072024-01-10T17:55:29.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.209161567Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11082024-01-10T17:55:29.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.209195603Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11092024-01-10T17:55:29.438Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.209918423Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"0","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11102024-01-10T17:55:29.438Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.209977552Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"0","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11112024-01-10T17:55:29.438Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.210140206Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"1","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11122024-01-10T17:55:29.439Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.210186127Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"1","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11132024-01-10T17:55:29.439Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.210826561Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"0","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11142024-01-10T17:55:29.439Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.210982572Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"2","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11152024-01-10T17:55:29.439Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.211033707Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"2","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11162024-01-10T17:55:29.439Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.211114657Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"1","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11172024-01-10T17:55:29.439Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.211851893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"0","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11182024-01-10T17:55:29.439Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.211933185Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"0","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11192024-01-10T17:55:29.439Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.211979131Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11202024-01-10T17:55:29.439Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.212050887Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"1","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11212024-01-10T17:55:29.439Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.212161159Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"1","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11222024-01-10T17:55:29.440Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.21220867Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11232024-01-10T17:55:29.440Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.212397792Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"2","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11242024-01-10T17:55:29.440Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.21304653Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"2","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11252024-01-10T17:55:29.440Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213098227Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"2","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11262024-01-10T17:55:29.440Z{"msg":"[0]R flush_numbers[0..12]: [12, 12, 9, 10, 10, 13, 1, 9, 8, 10, 12, 9]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213123705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11272024-01-10T17:55:29.440Z{"msg":"[0]R generation[0..12]: [8, 8, 6, 7, 7, 9, 1, 6, 6, 7, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213146652Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11282024-01-10T17:55:29.440Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213188814Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11292024-01-10T17:55:29.440Z{"msg":"[1]R flush_numbers[0..12]: [12, 12, 9, 10, 10, 13, 1, 9, 8, 10, 12, 9]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213212899Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11302024-01-10T17:55:29.440Z{"msg":"[1]R generation[0..12]: [8, 8, 6, 7, 7, 9, 1, 6, 6, 7, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213235469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11312024-01-10T17:55:29.441Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213258973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11322024-01-10T17:55:29.442Z{"msg":"[2]R flush_numbers[0..12]: [12, 12, 9, 10, 10, 13, 1, 9, 8, 10, 12, 9]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
11332024-01-10T17:55:29.442Z,"time":"2024-01-10T17:55:29.213312878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11342024-01-10T17:55:29.442Z{"msg":"[2]R generation[0..12]: [8, 8, 6, 7, 7, 9, 1, 6, 6, 7, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213422874Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11352024-01-10T17:55:29.443Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213451637Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11362024-01-10T17:55:29.443Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213517235Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11372024-01-10T17:55:29.443Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213562907Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11382024-01-10T17:55:29.443Z{"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213619289Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11392024-01-10T17:55:29.443Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213661194Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"":"downstairs","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11402024-01-10T17:55:29.445Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213702686Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11412024-01-10T17:55:29.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213759535Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"0","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11422024-01-10T17:55:29.446Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213802353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"1","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11432024-01-10T17:55:29.446Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213857568Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"client":"2","":"downstairs","session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11442024-01-10T17:55:29.446Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213899975Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11452024-01-10T17:55:29.446Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213951989Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11462024-01-10T17:55:29.446Z{"msg":"bf811484-668a-4729-afe8-511778dbdce6 is now active with session: 40a26109-29dd-4683-ab43-f181ce855cd3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.213992843Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11472024-01-10T17:55:29.446Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.214050892Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195,"session_id":"40a26109-29dd-4683-ab43-f181ce855cd3"}
11482024-01-10T17:55:29.446Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:29.214073669Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1195}
11492024-01-10T17:55:29.447ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11502024-01-10T17:55:29.447ZLoading write count information from file "/tmp/test_repair_verify.data"
11512024-01-10T17:55:29.447ZRun Repair workload
11522024-01-10T17:55:29.447Z01/30 Write block 114 len 32768 data: 3 3 3 2 2 2 2 2
11532024-01-10T17:55:29.447Z02/30 Flush
11542024-01-10T17:55:29.447Z03/30 Write block 237 len 16384 data: 2 2 2 2
11552024-01-10T17:55:29.447Z04/30 Read block 208 len 20480
11562024-01-10T17:55:29.447Z05/30 Read block 35 len 40960
11572024-01-10T17:55:29.447Z06/30 Read block 322 len 32768
11582024-01-10T17:55:29.447Z07/30 Read block 227 len 40960
11592024-01-10T17:55:29.448Z08/30 Flush
11602024-01-10T17:55:29.448Z09/30 Write block 44 len 8192 data: 2 2
11612024-01-10T17:55:29.448Z10/30 Flush
11622024-01-10T17:55:29.448Z11/30 Write block 503 len 20480 data: 4 3 3 3 4
11632024-01-10T17:55:29.448Z12/30 Read block 215 len 12288
11642024-01-10T17:55:29.448Z13/30 Read block 144 len 8192
11652024-01-10T17:55:29.448Z14/30 Read block 316 len 40960
11662024-01-10T17:55:29.454Z15/30 Read block 408 len 4096
11672024-01-10T17:55:29.454Z16/30 Read block 323 len 28672
11682024-01-10T17:55:29.454Z17/30 Write block 109 len 4096 data: 3
11692024-01-10T17:55:29.454Z18/30 Write block 327 len 8192 data: 3 3
11702024-01-10T17:55:29.454Z19/30 Read block 332 len 36864
11712024-01-10T17:55:29.454Z20/30 Read block 368 len 16384
11722024-01-10T17:55:29.455Z21/30 Flush
11732024-01-10T17:55:29.455Z22/30 Write block 531 len 32768 data: 2 2 2 2 2 3 3 3
11742024-01-10T17:55:29.455Z23/30 Write block 509 len 40960 data: 3 3 3 3 4 3 3 4 4 3
11752024-01-10T17:55:29.455Z24/30 Read block 331 len 16384
11762024-01-10T17:55:29.455Z25/30 Write block 518 len 12288 data: 4 3 3
11772024-01-10T17:55:29.460Z26/30 Write block 509 len 8192 data: 4 4
11782024-01-10T17:55:29.460Z27/30 Read block 103 len 36864
11792024-01-10T17:55:29.461Z28/30 Read block 310 len 12288
11802024-01-10T17:55:29.466Z29/30 Write block 298 len 28672 data: 3 3 3 3 2 2 3
11812024-01-10T17:55:29.466Z30/30 Write block 335 len 36864 data: 3 3 3 3 3 3 4 4 4
11822024-01-10T17:55:29.466Z----------------------------------------------------------------
11832024-01-10T17:55:29.468Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:28
11842024-01-10T17:55:29.469ZStates: Active Active Active
11852024-01-10T17:55:29.469ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11862024-01-10T17:55:29.469Z 3 Acked 1002 Write 4 Done Done Sent false
11872024-01-10T17:55:29.469Z 4 Acked 1003 Read 5 Done Done Sent false
11882024-01-10T17:55:29.469Z 5 Acked 1004 Read 10 Done Done Sent false
11892024-01-10T17:55:29.469Z 6 Acked 1005 Read 8 Done Done Sent false
11902024-01-10T17:55:29.469Z 7 Acked 1006 Read 10 Done Done Sent false
11912024-01-10T17:55:29.469Z 8 Acked 1007 Flush 0 Done Done Sent false
11922024-01-10T17:55:29.469Z 9 Acked 1008 Write 2 Done Done Sent false
11932024-01-10T17:55:29.469Z 10 Acked 1009 Flush 0 Done Done Sent false
11942024-01-10T17:55:29.469Z 11 Acked 1010 Write 5 Done Done Sent false
11952024-01-10T17:55:29.469Z 12 Acked 1011 Read 3 Done Done Sent false
11962024-01-10T17:55:29.469Z 13 Acked 1012 Read 2 Done Done Sent false
11972024-01-10T17:55:29.469Z 14 Acked 1013 Read 10 Done Done Sent false
11982024-01-10T17:55:29.469Z 15 Acked 1014 Read 1 Done Done Sent false
11992024-01-10T17:55:29.469Z 16 Acked 1015 Read 7 Done Done Sent false
12002024-01-10T17:55:29.469Z 17 Acked 1016 Write 1 Done Done Sent false
12012024-01-10T17:55:29.469Z 18 Acked 1017 Write 2 Done Done Sent false
12022024-01-10T17:55:29.469Z 19 Acked 1018 Read 9 Done Done Sent false
12032024-01-10T17:55:29.469Z 20 Acked 1019 Read 4 Done Done Sent false
12042024-01-10T17:55:29.469Z 21 Acked 1020 Flush 0 Done Done Sent false
12052024-01-10T17:55:29.469Z 22 Acked 1021 Write 8 Done Done Sent false
12062024-01-10T17:55:29.473Z 23 Acked 1022 Write 10 Done Done Sent false
12072024-01-10T17:55:29.473Z 24 Acked 1023 Read 4 Done Done Sent false
12082024-01-10T17:55:29.473Z 25 Acked 1024 Write 3 Done Done Sent false
12092024-01-10T17:55:29.473Z 26 Acked 1025 Write 2 Done Done Sent false
12102024-01-10T17:55:29.473Z 27 Acked 1026 Read 9 Done Done Sent false
12112024-01-10T17:55:29.473Z 28 Acked 1027 Read 3 Done Done Sent false
12122024-01-10T17:55:29.473Z 29 Acked 1028 Write 7 Done Done Sent false
12132024-01-10T17:55:29.473Z 30 Acked 1029 Write 9 Sent Sent Sent false
12142024-01-10T17:55:29.473Z STATES DS:0 DS:1 DS:2 TOTAL
12152024-01-10T17:55:29.473Z New 0 0 0 0
12162024-01-10T17:55:29.473Z Sent 1 1 28 30
12172024-01-10T17:55:29.474Z Done 27 27 0 54
12182024-01-10T17:55:29.474Z Skipped 0 0 0 0
12192024-01-10T17:55:29.474Z Error 0 0 0 0
12202024-01-10T17:55:29.474ZLast Flush: 1020 1020 1001
12212024-01-10T17:55:29.474ZDownstairs last five completed: 1001 1000
12222024-01-10T17:55:29.474ZUpstairs last five completed: 30 29 28 27 26
12232024-01-10T17:55:29.474ZWrote out file "/tmp/test_repair_verify.data"
12242024-01-10T17:55:29.478Z
12252024-01-10T17:55:29.478ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12262024-01-10T17:55:29.502ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12272024-01-10T17:55:29.502Z 0 000-019 8 8 8 12 12 12 F F F
12282024-01-10T17:55:29.502Z 1 020-039 8 8 8 12 12 12 F F F
12292024-01-10T17:55:29.502Z 2 040-059 10 10 6 16 16 9 F F F <---
12302024-01-10T17:55:29.502Z 3 060-079 7 7 7 10 10 10 F F F
12312024-01-10T17:55:29.502Z 4 080-099 7 7 7 10 10 10 F F F
12322024-01-10T17:55:29.507Z 5 100-119 10 10 10 17 17 14 F F F <---
12332024-01-10T17:55:29.509Z 6 120-139 10 10 10 14 14 14 F F F
12342024-01-10T17:55:29.509Z 7 140-159 6 6 6 9 9 9 F F F
12352024-01-10T17:55:29.509Z 8 160-179 6 6 6 8 8 8 F F F
12362024-01-10T17:55:29.509Z 9 180-199 7 7 7 10 10 10 F F F
12372024-01-10T17:55:29.509Z 10 200-219 8 8 8 12 12 12 F F F
12382024-01-10T17:55:29.509Z 11 220-239 10 10 6 15 15 9 F F F <---
12392024-01-10T17:55:29.509Z 12 240-259 10 10 8 15 15 12 F F F <---
12402024-01-10T17:55:29.509Z 13 260-279 4 4 4 6 6 6 F F F
12412024-01-10T17:55:29.509Z 14 280-299 9 9 9 13 13 13 T T F <---
12422024-01-10T17:55:29.509Z 15 300-319 9 9 9 13 13 13 T T F <---
12432024-01-10T17:55:29.509Z 16 320-339 10 10 8 17 17 12 T T F <---
12442024-01-10T17:55:29.510Z 17 340-359 8 8 8 12 12 12 T T F <---
12452024-01-10T17:55:29.510Z 18 360-379 1 1 1 1 1 1 F F F
12462024-01-10T17:55:29.510Z 19 380-399 2 2 2 4 4 4 F F F
12472024-01-10T17:55:29.510Z 20 400-419 5 5 5 7 7 7 F F F
12482024-01-10T17:55:29.510Z 21 420-439 7 7 7 10 10 10 F F F
12492024-01-10T17:55:29.510Z 22 440-459 8 8 8 11 11 11 F F F
12502024-01-10T17:55:29.510Z 23 460-479 4 4 4 6 6 6 F F F
12512024-01-10T17:55:29.510Z 24 480-499 8 8 8 12 12 12 F F F
12522024-01-10T17:55:29.510Z 25 500-519 10 10 8 17 17 12 T T F <---
12532024-01-10T17:55:29.510Z 26 520-539 8 8 8 12 12 12 T T F <---
12542024-01-10T17:55:29.510Z 27 540-559 9 9 9 13 13 13 F F F
12552024-01-10T17:55:29.511Z 28 560-579 8 8 8 11 11 11 F F F
12562024-01-10T17:55:29.511Z 29 580-599 7 7 7 10 10 10 F F F
12572024-01-10T17:55:29.511ZMax gen: 10, Max flush: 17
12582024-01-10T17:55:29.511ZError: Difference in extent metadata found!
12592024-01-10T17:55:29.512ZA Difference in extent metadata is expected here
12602024-01-10T17:55:29.512ZOn loop 5
12612024-01-10T17:55:29.512Z
12622024-01-10T17:55:29.938ZVerifying data now
12632024-01-10T17:55:30.551ZNo change after verify
12642024-01-10T17:55:30.551ZLoop: 5 Downstairs dump after verify (and repair):
12652024-01-10T17:55:30.573ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12662024-01-10T17:55:30.573Z 0 000-019 8 8 8 12 12 12 F F F
12672024-01-10T17:55:30.573Z 1 020-039 8 8 8 12 12 12 F F F
12682024-01-10T17:55:30.573Z 2 040-059 10 10 10 16 16 16 F F F
12692024-01-10T17:55:30.573Z 3 060-079 7 7 7 10 10 10 F F F
12702024-01-10T17:55:30.573Z 4 080-099 7 7 7 10 10 10 F F F
12712024-01-10T17:55:30.579Z 5 100-119 10 10 10 17 17 17 F F F
12722024-01-10T17:55:30.579Z 6 120-139 10 10 10 14 14 14 F F F
12732024-01-10T17:55:30.579Z 7 140-159 6 6 6 9 9 9 F F F
12742024-01-10T17:55:30.579Z 8 160-179 6 6 6 8 8 8 F F F
12752024-01-10T17:55:30.579Z 9 180-199 7 7 7 10 10 10 F F F
12762024-01-10T17:55:30.579Z 10 200-219 8 8 8 12 12 12 F F F
12772024-01-10T17:55:30.579Z 11 220-239 10 10 10 15 15 15 F F F
12782024-01-10T17:55:30.579Z 12 240-259 10 10 10 15 15 15 F F F
12792024-01-10T17:55:30.579Z 13 260-279 4 4 4 6 6 6 F F F
12802024-01-10T17:55:30.579Z 14 280-299 11 11 11 18 18 18 F F F
12812024-01-10T17:55:30.580Z 15 300-319 11 11 11 18 18 18 F F F
12822024-01-10T17:55:30.580Z 16 320-339 11 11 11 18 18 18 F F F
12832024-01-10T17:55:30.580Z 17 340-359 11 11 11 18 18 18 F F F
12842024-01-10T17:55:30.580Z 18 360-379 1 1 1 1 1 1 F F F
12852024-01-10T17:55:30.580Z 19 380-399 2 2 2 4 4 4 F F F
12862024-01-10T17:55:30.580Z 20 400-419 5 5 5 7 7 7 F F F
12872024-01-10T17:55:30.580Z 21 420-439 7 7 7 10 10 10 F F F
12882024-01-10T17:55:30.580Z 22 440-459 8 8 8 11 11 11 F F F
12892024-01-10T17:55:30.580Z 23 460-479 4 4 4 6 6 6 F F F
12902024-01-10T17:55:30.580Z 24 480-499 8 8 8 12 12 12 F F F
12912024-01-10T17:55:30.580Z 25 500-519 11 11 11 18 18 18 F F F
12922024-01-10T17:55:30.580Z 26 520-539 11 11 11 18 18 18 F F F
12932024-01-10T17:55:30.581Z 27 540-559 9 9 9 13 13 13 F F F
12942024-01-10T17:55:30.581Z 28 560-579 8 8 8 11 11 11 F F F
12952024-01-10T17:55:30.581Z 29 580-599 7 7 7 10 10 10 F F F
12962024-01-10T17:55:30.581ZMax gen: 11, Max flush: 18
12972024-01-10T17:55:30.582Z
12982024-01-10T17:55:30.582ZBegin loop 6
12992024-01-10T17:55:30.582ZDownstairs to restart: 1
13002024-01-10T17:55:30.609Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.419694924Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203}
13012024-01-10T17:55:30.609Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.420261982Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203}
13022024-01-10T17:55:30.619Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.420319633Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203}
13032024-01-10T17:55:30.619Z{"msg":"Crucible 8ce38cb9-9ae7-44d0-83d5-eaffc49c1b30 has session id: 4b641551-f2eb-4650-bca2-0ea6c7205fdf","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.42039035Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13042024-01-10T17:55:30.619Z{"msg":"Crucible runtime is spawned
13052024-01-10T17:55:30.619ZUpstairs opts: Upstairs UUID: 8ce38cb9-9ae7-44d0-83d5-eaffc49c1b30, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.4204921Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13062024-01-10T17:55:30.619Z{"msg":"Crucible stats registered with UUID: 8ce38cb9-9ae7-44d0-83d5-eaffc49c1b30","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.420556344Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13072024-01-10T17:55:30.620Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.420582468Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203}
13082024-01-10T17:55:30.620Z{"msg":"8ce38cb9-9ae7-44d0-83d5-eaffc49c1b30 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.42086812Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13092024-01-10T17:55:30.620Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.420930276Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"0","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13102024-01-10T17:55:30.620Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421031713Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"0","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13112024-01-10T17:55:30.620Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421118358Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"1","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13122024-01-10T17:55:30.621Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421161635Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"1","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13132024-01-10T17:55:30.621Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421198913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"2","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13142024-01-10T17:55:30.621Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421229813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"2","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13152024-01-10T17:55:30.621Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421252808Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13162024-01-10T17:55:30.621Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421283432Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13172024-01-10T17:55:30.621Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421375058Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13182024-01-10T17:55:30.622Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421523795Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13192024-01-10T17:55:30.622Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421659865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13202024-01-10T17:55:30.623Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.421798588Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13212024-01-10T17:55:30.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.423005388Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"0","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13222024-01-10T17:55:30.623Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.423087129Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"0","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13232024-01-10T17:55:30.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.423182485Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"1","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13242024-01-10T17:55:30.623Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.423263525Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"1","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13252024-01-10T17:55:30.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.423291887Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"2","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13262024-01-10T17:55:30.624Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.423313726Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"2","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13272024-01-10T17:55:30.624Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.424216792Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"2","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13282024-01-10T17:55:30.624Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.424351201Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"0","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13292024-01-10T17:55:30.624Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.424433817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"1","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13302024-01-10T17:55:30.624Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425183888Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"2","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13312024-01-10T17:55:30.624Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.42525529Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"2","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13322024-01-10T17:55:30.625Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425300322Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13332024-01-10T17:55:30.625Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425393427Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"1","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13342024-01-10T17:55:30.625Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425436253Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"1","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13352024-01-10T17:55:30.625Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425497619Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13362024-01-10T17:55:30.625Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425522728Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"0","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13372024-01-10T17:55:30.625Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425544781Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"0","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13382024-01-10T17:55:30.625Z{"msg":"[0]R flush_numbers[0..12]: [12, 12, 16, 10, 10, 17, 14, 9, 8, 10, 12, 15]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425585942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13392024-01-10T17:55:30.625Z{"msg":"[0]R generation[0..12]: [8, 8, 10, 7, 7, 10, 10, 6, 6, 7, 8, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425610153Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13402024-01-10T17:55:30.625Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425633783Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13412024-01-10T17:55:30.625Z{"msg":"[1]R flush_numbers[0..12]: [12, 12, 16, 10, 10, 17, 14, 9, 8, 10, 12, 15]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425656712Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13422024-01-10T17:55:30.625Z{"msg":"[1]R generation[0..12]: [8, 8, 10, 7, 7, 10, 10, 6, 6, 7, 8, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425678962Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13432024-01-10T17:55:30.625Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425720469Z","hostname":"Wait for a query_work_queue command to finish before sending IO
13442024-01-10T17:55:30.625Zip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13452024-01-10T17:55:30.626Z{"msg":"[2]R flush_numbers[0..12]: [12, 12, 16, 10, 10, 17, 14, 9, 8, 10, 12, 15]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425747546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13462024-01-10T17:55:30.626Z{"msg":"[2]R generation[0..12]: [8, 8, 10, 7, 7, 10, 10, 6, 6, 7, 8, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425793837Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13472024-01-10T17:55:30.626Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.42582207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13482024-01-10T17:55:30.626Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425843899Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13492024-01-10T17:55:30.626Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425876309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13502024-01-10T17:55:30.626Z{"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.425951277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13512024-01-10T17:55:30.626Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.426001051Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"":"downstairs","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13522024-01-10T17:55:30.626Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.426037862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13532024-01-10T17:55:30.626Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.426107198Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"0","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13542024-01-10T17:55:30.626Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.426140804Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"1","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13552024-01-10T17:55:30.626Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.426182868Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"client":"2","":"downstairs","session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13562024-01-10T17:55:30.626Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.426206307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13572024-01-10T17:55:30.626Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.426265796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13582024-01-10T17:55:30.626Z{"msg":"8ce38cb9-9ae7-44d0-83d5-eaffc49c1b30 is now active with session: 4b641551-f2eb-4650-bca2-0ea6c7205fdf","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.42630072Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13592024-01-10T17:55:30.627Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.426341667Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203,"session_id":"4b641551-f2eb-4650-bca2-0ea6c7205fdf"}
13602024-01-10T17:55:30.627Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:30.42637037Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1203}
13612024-01-10T17:55:30.627ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13622024-01-10T17:55:30.627ZLoading write count information from file "/tmp/test_repair_verify.data"
13632024-01-10T17:55:30.627ZRun Repair workload
13642024-01-10T17:55:30.627Z01/30 Write block 70 len 4096 data: 2
13652024-01-10T17:55:30.631Z02/30 Write block 97 len 28672 data: 2 2 2 2 2 2 2
13662024-01-10T17:55:30.631Z03/30 Write block 137 len 20480 data: 2 2 2 3 4
13672024-01-10T17:55:30.632Z04/30 Write block 357 len 32768 data: 2 3 2 2 2 2 2 2
13682024-01-10T17:55:30.632Z05/30 Write block 236 len 40960 data: 2 3 3 3 3 2 2 2 2 2
13692024-01-10T17:55:30.632Z06/30 Read block 4 len 40960
13702024-01-10T17:55:30.632Z07/30 Write block 439 len 36864 data: 3 3 3 3 3 4 5 5 3
13712024-01-10T17:55:30.632Z08/30 Write block 512 len 12288 data: 4 5 4
13722024-01-10T17:55:30.632Z09/30 Write block 66 len 4096 data: 2
13732024-01-10T17:55:30.632Z10/30 Flush
13742024-01-10T17:55:30.637Z11/30 Write block 274 len 16384 data: 3 3 3 2
13752024-01-10T17:55:30.637Z12/30 Read block 29 len 20480
13762024-01-10T17:55:30.641Z13/30 Write block 441 len 32768 data: 4 4 4 5 6 6 4 3
13772024-01-10T17:55:30.641Z14/30 Write block 171 len 32768 data: 2 2 2 2 2 2 2 2
13782024-01-10T17:55:30.642Z15/30 Read block 262 len 36864
13792024-01-10T17:55:30.648Z16/30 Write block 375 len 28672 data: 2 2 2 2 2 3 3
13802024-01-10T17:55:30.648Z17/30 Write block 334 len 24576 data: 2 4 4 4 4 4
13812024-01-10T17:55:30.648Z18/30 Write block 484 len 20480 data: 2 2 2 2 2
13822024-01-10T17:55:30.648Z19/30 Read block 102 len 28672
13832024-01-10T17:55:30.655Z20/30 Write block 394 len 32768 data: 2 2 2 2 2 2 2 2
13842024-01-10T17:55:30.655Z21/30 Read block 8 len 20480
13852024-01-10T17:55:30.664Z22/30 Write block 115 len 32768 data: 4 4 3 3 3 3 3 2
13862024-01-10T17:55:30.664Z23/30 Read block 402 len 4096
13872024-01-10T17:55:30.665Z24/30 Write block 337 len 12288 data: 5 5 5
13882024-01-10T17:55:30.665Z25/30 Read block 69 len 12288
13892024-01-10T17:55:30.665Z26/30 Write block 589 len 32768 data: 2 2 3 3 3 3 3 3
13902024-01-10T17:55:30.665Z27/30 Read block 288 len 12288
13912024-01-10T17:55:31.315Z28/30 Read block 417 len 4096
13922024-01-10T17:55:31.315Z29/30 Write block 576 len 32768 data: 2 2 2 2 2 2 3 3
13932024-01-10T17:55:31.315Z30/30 Write block 162 len 20480 data: 4 4 4 4 3
13942024-01-10T17:55:31.315Z----------------------------------------------------------------
13952024-01-10T17:55:31.315Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
13962024-01-10T17:55:31.316ZStates: Active Active Active
13972024-01-10T17:55:31.316ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
13982024-01-10T17:55:31.316Z 1 Acked 1000 Write 1 Done Sent Done false
13992024-01-10T17:55:31.316Z 2 Acked 1001 Write 7 Done Sent Done false
14002024-01-10T17:55:31.316Z 3 Acked 1002 Write 5 Done Sent Done false
14012024-01-10T17:55:31.316Z 4 Acked 1003 Write 8 Done Sent Done false
14022024-01-10T17:55:31.316Z 5 Acked 1004 Write 10 Done Sent Done false
14032024-01-10T17:55:31.316Z 6 Acked 1005 Read 10 Done Sent Done false
14042024-01-10T17:55:31.316Z 7 Acked 1006 Write 9 Done Sent Done false
14052024-01-10T17:55:31.316Z 8 Acked 1007 Write 3 Done Sent Done false
14062024-01-10T17:55:31.316Z 9 Acked 1008 Write 1 Done Sent Done false
14072024-01-10T17:55:31.316Z 10 Acked 1009 Flush 0 Done Sent Done false
14082024-01-10T17:55:31.316Z 11 Acked 1010 Write 4 Done Sent Done false
14092024-01-10T17:55:31.316Z 12 Acked 1011 Read 5 Done Sent Done false
14102024-01-10T17:55:31.316Z 13 Acked 1012 Write 8 Done Sent Done false
14112024-01-10T17:55:31.316Z 14 Acked 1013 Write 8 Done Sent Done false
14122024-01-10T17:55:31.316Z 15 Acked 1014 Read 9 Done Sent Done false
14132024-01-10T17:55:31.316Z 16 Acked 1015 Write 7 Done Sent Done false
14142024-01-10T17:55:31.316Z 17 Acked 1016 Write 6 Done Sent Done false
14152024-01-10T17:55:31.316Z 18 Acked 1017 Write 5 Done Sent Done false
14162024-01-10T17:55:31.316Z 19 Acked 1018 Read 7 Done Sent Done false
14172024-01-10T17:55:31.316Z 20 Acked 1019 Write 8 Done Sent Done false
14182024-01-10T17:55:31.316Z 21 Acked 1020 Read 5 Done Sent Done false
14192024-01-10T17:55:31.317Z 22 Acked 1021 Write 8 Done Sent Done false
14202024-01-10T17:55:31.317Z 23 Acked 1022 Read 1 Done Sent Done false
14212024-01-10T17:55:31.317Z 24 Acked 1023 Write 3 Done Sent Done false
14222024-01-10T17:55:31.317Z 25 Acked 1024 Read 3 Done Sent Done false
14232024-01-10T17:55:31.317Z 26 Acked 1025 Write 8 Done Sent Done false
14242024-01-10T17:55:31.317Z 27 Acked 1026 Read 3 Done Sent Done false
14252024-01-10T17:55:31.318Z 28 Acked 1027 Read 1 Done Sent Done false
14262024-01-10T17:55:31.318Z 29 Acked 1028 Write 8 Sent Sent Sent false
14272024-01-10T17:55:31.318Z 30 Acked 1029 Write 5 Sent Sent Sent false
14282024-01-10T17:55:31.318Z STATES DS:0 DS:1 DS:2 TOTAL
14292024-01-10T17:55:31.318Z New 0 0 0 0
14302024-01-10T17:55:31.318Z Sent 2 30 2 34
14312024-01-10T17:55:31.318Z Done 28 0 28 56
14322024-01-10T17:55:31.318Z Skipped 0 0 0 0
14332024-01-10T17:55:31.318Z Error 0 0 0 0
14342024-01-10T17:55:31.318ZLast Flush: 1009 0 1009
14352024-01-10T17:55:31.318ZDownstairs last five completed:
14362024-01-10T17:55:31.318ZUpstairs last five completed: 30 29 28 27 26
14372024-01-10T17:55:31.318ZWrote out file "/tmp/test_repair_verify.data"
14382024-01-10T17:55:31.322Z
14392024-01-10T17:55:31.322ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14402024-01-10T17:55:31.322ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14412024-01-10T17:55:31.322Z 0 000-019 8 8 8 12 12 12 F F F
14422024-01-10T17:55:31.322Z 1 020-039 8 8 8 12 12 12 F F F
14432024-01-10T17:55:31.322Z 2 040-059 10 10 10 16 16 16 F F F
14442024-01-10T17:55:31.322Z 3 060-079 12 7 12 19 10 19 F F F <---
14452024-01-10T17:55:31.322Z 4 080-099 12 7 12 19 10 19 F F F <---
14462024-01-10T17:55:31.324Z 5 100-119 12 10 12 19 17 19 T F T <---
14472024-01-10T17:55:31.324Z 6 120-139 12 10 12 19 14 19 T F T <---
14482024-01-10T17:55:31.325Z 7 140-159 12 6 12 19 9 19 F F F <---
14492024-01-10T17:55:31.325Z 8 160-179 6 6 6 8 8 8 T F T <---
14502024-01-10T17:55:31.325Z 9 180-199 7 7 7 10 10 10 F F F
14512024-01-10T17:55:31.325Z 10 200-219 8 8 8 12 12 12 F F F
14522024-01-10T17:55:31.325Z 11 220-239 12 10 12 19 15 19 F F F <---
14532024-01-10T17:55:31.325Z 12 240-259 12 10 12 19 15 19 F F F <---
14542024-01-10T17:55:31.325Z 13 260-279 4 4 4 6 6 6 T F T <---
14552024-01-10T17:55:31.325Z 14 280-299 11 11 11 18 18 18 F F F
14562024-01-10T17:55:31.325Z 15 300-319 11 11 11 18 18 18 F F F
14572024-01-10T17:55:31.325Z 16 320-339 11 11 11 18 18 18 T F T <---
14582024-01-10T17:55:31.325Z 17 340-359 12 11 12 19 18 19 F F F <---
14592024-01-10T17:55:31.325Z 18 360-379 12 1 12 19 1 19 T F T <---
14602024-01-10T17:55:31.325Z 19 380-399 2 2 2 4 4 4 T F T <---
14612024-01-10T17:55:31.325Z 20 400-419 5 5 5 7 7 7 T F T <---
14622024-01-10T17:55:31.325Z 21 420-439 12 7 12 19 10 19 F F F <---
14632024-01-10T17:55:31.325Z 22 440-459 12 8 12 19 11 19 T F T <---
14642024-01-10T17:55:31.325Z 23 460-479 4 4 4 6 6 6 F F F
14652024-01-10T17:55:31.325Z 24 480-499 8 8 8 12 12 12 T F T <---
14662024-01-10T17:55:31.325Z 25 500-519 12 11 12 19 18 19 F F F <---
14672024-01-10T17:55:31.325Z 26 520-539 11 11 11 18 18 18 F F F
14682024-01-10T17:55:31.325Z 27 540-559 9 9 9 13 13 13 F F F
14692024-01-10T17:55:31.325Z 28 560-579 8 8 8 11 11 11 T F T <---
14702024-01-10T17:55:31.326Z 29 580-599 7 7 7 10 10 10 T F T <---
14712024-01-10T17:55:31.326ZMax gen: 12, Max flush: 19
14722024-01-10T17:55:31.326ZError: Difference in extent metadata found!
14732024-01-10T17:55:31.326ZA Difference in extent metadata is expected here
14742024-01-10T17:55:31.326ZOn loop 6
14752024-01-10T17:55:31.326Z
14762024-01-10T17:55:31.327ZVerifying data now
14772024-01-10T17:55:36.544ZNo change after verify
14782024-01-10T17:55:36.544ZLoop: 6 Downstairs dump after verify (and repair):
14792024-01-10T17:55:36.565ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14802024-01-10T17:55:36.565Z 0 000-019 8 8 8 12 12 12 F F F
14812024-01-10T17:55:36.565Z 1 020-039 8 8 8 12 12 12 F F F
14822024-01-10T17:55:36.565Z 2 040-059 10 10 10 16 16 16 F F F
14832024-01-10T17:55:36.565Z 3 060-079 12 12 12 19 19 19 F F F
14842024-01-10T17:55:36.565Z 4 080-099 12 12 12 19 19 19 F F F
14852024-01-10T17:55:36.565Z 5 100-119 13 13 13 20 20 20 F F F
14862024-01-10T17:55:36.570Z 6 120-139 13 13 13 20 20 20 F F F
14872024-01-10T17:55:36.571Z 7 140-159 12 12 12 19 19 19 F F F
14882024-01-10T17:55:36.571Z 8 160-179 13 13 13 20 20 20 F F F
14892024-01-10T17:55:36.571Z 9 180-199 7 7 7 10 10 10 F F F
14902024-01-10T17:55:36.571Z 10 200-219 8 8 8 12 12 12 F F F
14912024-01-10T17:55:36.571Z 11 220-239 12 12 12 19 19 19 F F F
14922024-01-10T17:55:36.571Z 12 240-259 12 12 12 19 19 19 F F F
14932024-01-10T17:55:36.571Z 13 260-279 13 13 13 20 20 20 F F F
14942024-01-10T17:55:36.571Z 14 280-299 11 11 11 18 18 18 F F F
14952024-01-10T17:55:36.571Z 15 300-319 11 11 11 18 18 18 F F F
14962024-01-10T17:55:36.571Z 16 320-339 13 13 13 20 20 20 F F F
14972024-01-10T17:55:36.571Z 17 340-359 12 12 12 19 19 19 F F F
14982024-01-10T17:55:36.571Z 18 360-379 13 13 13 20 20 20 F F F
14992024-01-10T17:55:36.572Z 19 380-399 13 13 13 20 20 20 F F F
15002024-01-10T17:55:36.572Z 20 400-419 13 13 13 20 20 20 F F F
15012024-01-10T17:55:36.572Z 21 420-439 12 12 12 19 19 19 F F F
15022024-01-10T17:55:36.572Z 22 440-459 13 13 13 20 20 20 F F F
15032024-01-10T17:55:36.572Z 23 460-479 4 4 4 6 6 6 F F F
15042024-01-10T17:55:36.572Z 24 480-499 13 13 13 20 20 20 F F F
15052024-01-10T17:55:36.572Z 25 500-519 12 12 12 19 19 19 F F F
15062024-01-10T17:55:36.572Z 26 520-539 11 11 11 18 18 18 F F F
15072024-01-10T17:55:36.572Z 27 540-559 9 9 9 13 13 13 F F F
15082024-01-10T17:55:36.572Z 28 560-579 13 13 13 20 20 20 F F F
15092024-01-10T17:55:36.572Z 29 580-599 13 13 13 20 20 20 F F F
15102024-01-10T17:55:36.572ZMax gen: 13, Max flush: 20
15112024-01-10T17:55:36.572Z
15122024-01-10T17:55:36.573ZBegin loop 7
15132024-01-10T17:55:36.574ZDownstairs to restart: 0
15142024-01-10T17:55:36.597Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.409226996Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211}
15152024-01-10T17:55:36.601Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.409730823Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211}
15162024-01-10T17:55:36.601Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.409795858Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211}
15172024-01-10T17:55:36.601Z{"msg":"Crucible 4c750989-f1c8-4193-a89b-f8af0dfa8129 has session id: 22f0fad5-53c0-4827-a069-e4a343ff7d87","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.409838238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15182024-01-10T17:55:36.602Z{"msg":"Upstairs opts: Upstairs UUID: 4c750989-f1c8-4193-a89b-f8af0dfa8129, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.40989635Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15192024-01-10T17:55:36.602Z{"msg":"Crucible stats registered with UUID: 4c750989-f1c8-4193-a89b-f8af0dfa8129","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.409995646Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15202024-01-10T17:55:36.602ZCrucible runtime is spawned
15212024-01-10T17:55:36.602Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.410351699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211}
15222024-01-10T17:55:36.602Z{"msg":"4c750989-f1c8-4193-a89b-f8af0dfa8129 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.41048884Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15232024-01-10T17:55:36.602Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.410578247Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"0","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15242024-01-10T17:55:36.602Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.41060532Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"0","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15252024-01-10T17:55:36.602Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.410814541Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"1","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15262024-01-10T17:55:36.602Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.410956711Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"1","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15272024-01-10T17:55:36.602Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.411072428Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"2","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15282024-01-10T17:55:36.604Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.411184277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"2","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15292024-01-10T17:55:36.604Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.41124161Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15302024-01-10T17:55:36.604Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.411371626Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15312024-01-10T17:55:36.604Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.411418023Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15322024-01-10T17:55:36.604Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.411461978Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15332024-01-10T17:55:36.604Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.41150383Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15342024-01-10T17:55:36.604Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.411715705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15352024-01-10T17:55:36.604Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.412085789Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"2","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15362024-01-10T17:55:36.604Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.412156571Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"2","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15372024-01-10T17:55:36.604Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.412257521Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"1","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15382024-01-10T17:55:36.605Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.412302738Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"1","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15392024-01-10T17:55:36.611Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.413203738Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"2","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15402024-01-10T17:55:36.611Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.413349847Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"1","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15412024-01-10T17:55:36.611Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.413505648Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"0","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15422024-01-10T17:55:36.611Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.413552551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"0","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15432024-01-10T17:55:36.611Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.414363127Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"1","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15442024-01-10T17:55:36.611Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.414421887Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"1","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15452024-01-10T17:55:36.611Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.414446978Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15462024-01-10T17:55:36.611Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.414494693Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"2","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15472024-01-10T17:55:36.611Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.414518491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"2","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15482024-01-10T17:55:36.612Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.414558371Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15492024-01-10T17:55:36.612Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.414581157Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"0","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15502024-01-10T17:55:36.612Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.415426669Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"0","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15512024-01-10T17:55:36.612Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.415501969Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"0","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15522024-01-10T17:55:36.612Z{"msg":"[0]R flush_numbers[0..12]: [12, 12, 16, 19, 19, 20, 20, 19, 20, 10, 12, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.41553185Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15532024-01-10T17:55:36.612Z{"msg":"[0]R generation[0..12]: [8, 8, 10, 12, 12, 13, 13, 12, 13, 7, 8, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.415574845Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15542024-01-10T17:55:36.612Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.415600518Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15552024-01-10T17:55:36.612ZWait for a query_work_queue command to finish before sending IO
15562024-01-10T17:55:36.612Z{"msg":"[1]R flush_numbers[0..12]: [12, 12, 16, 19, 19, 20, 20, 19, 20, 10, 12, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.41566552Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15572024-01-10T17:55:36.612Z{"msg":"[1]R generation[0..12]: [8, 8, 10, 12, 12, 13, 13, 12, 13, 7, 8, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.415736806Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15582024-01-10T17:55:36.612Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.415810505Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15592024-01-10T17:55:36.612Z{"msg":"[2]R flush_numbers[0..12]: [12, 12, 16, 19, 19, 20, 20, 19, 20, 10, 12, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.415874601Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15602024-01-10T17:55:36.612Z{"msg":"[2]R generation[0..12]: [8, 8, 10, 12, 12, 13, 13, 12, 13, 7, 8, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.415901299Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15612024-01-10T17:55:36.612Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.415965679Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15622024-01-10T17:55:36.612Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416046761Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15632024-01-10T17:55:36.612Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416084159Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15642024-01-10T17:55:36.612Z{"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416143119Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15652024-01-10T17:55:36.613Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416176201Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"":"downstairs","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15662024-01-10T17:55:36.613Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416245655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15672024-01-10T17:55:36.613Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416277388Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"0","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15682024-01-10T17:55:36.614Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416300106Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"1","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15692024-01-10T17:55:36.614Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416341742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"client":"2","":"downstairs","session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15702024-01-10T17:55:36.614Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15712024-01-10T17:55:36.614Z,"time":"2024-01-10T17:55:36.416377109Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15722024-01-10T17:55:36.614Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.41650146Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15732024-01-10T17:55:36.614Z{"msg":"4c750989-f1c8-4193-a89b-f8af0dfa8129 is now active with session: 22f0fad5-53c0-4827-a069-e4a343ff7d87","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416574386Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15742024-01-10T17:55:36.614Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416639791Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211,"session_id":"22f0fad5-53c0-4827-a069-e4a343ff7d87"}
15752024-01-10T17:55:36.614Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:36.416705235Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1211}
15762024-01-10T17:55:36.614ZLoading write count information from file "/tmp/test_repair_verify.data"
15772024-01-10T17:55:36.614ZRun Repair workload
15782024-01-10T17:55:36.614Z01/30 Write block 244 len 36864 data: 3 3 2 2 2 3 3 3 3
15792024-01-10T17:55:36.614Z02/30 Read block 24 len 32768
15802024-01-10T17:55:36.617Z03/30 Read block 407 len 40960
15812024-01-10T17:55:36.617Z04/30 Write block 467 len 8192 data: 3 3
15822024-01-10T17:55:36.618Z05/30 Read block 248 len 16384
15832024-01-10T17:55:36.618Z06/30 Write block 91 len 4096 data: 2
15842024-01-10T17:55:36.618Z07/30 Read block 439 len 20480
15852024-01-10T17:55:36.621Z08/30 Read block 288 len 4096
15862024-01-10T17:55:36.621Z09/30 Write block 463 len 32768 data: 3 3 3 3 4 4 2 2
15872024-01-10T17:55:36.625Z10/30 Read block 350 len 12288
15882024-01-10T17:55:36.625Z11/30 Read block 359 len 24576
15892024-01-10T17:55:36.630Z12/30 Read block 60 len 32768
15902024-01-10T17:55:36.634Z13/30 Read block 494 len 8192
15912024-01-10T17:55:36.634Z14/30 Read block 39 len 12288
15922024-01-10T17:55:36.634Z15/30 Write block 159 len 36864 data: 2 2 2 5 5 5 5 4 3
15932024-01-10T17:55:36.637Z16/30 Write block 7 len 40960 data: 2 2 2 2 2 4 4 4 4 4
15942024-01-10T17:55:36.637Z17/30 Write block 73 len 8192 data: 2 2
15952024-01-10T17:55:36.637Z18/30 Write block 194 len 8192 data: 2 2
15962024-01-10T17:55:36.637Z19/30 Flush
15972024-01-10T17:55:36.675Z20/30 Write block 565 len 28672 data: 3 2 2 2 2 2 2
15982024-01-10T17:55:36.678Z21/30 Write block 263 len 36864 data: 2 2 2 2 3 3 3 3 3
15992024-01-10T17:55:36.679Z22/30 Read block 377 len 24576
16002024-01-10T17:55:36.679Z23/30 Read block 577 len 12288
16012024-01-10T17:55:36.679Z24/30 Write block 419 len 24576 data: 4 4 4 3 2 2
16022024-01-10T17:55:36.679Z25/30 Write block 289 len 12288 data: 2 2 2
16032024-01-10T17:55:36.679Z26/30 Write block 194 len 24576 data: 3 3 2 2 2 3
16042024-01-10T17:55:36.679Z27/30 Read block 59 len 12288
16052024-01-10T17:55:36.679Z28/30 Read block 1 len 36864
16062024-01-10T17:55:36.680Z29/30 Write block 494 len 24576 data: 3 4 4 4 4 5
16072024-01-10T17:55:36.680Z30/30 Read block 309 len 28672
16082024-01-10T17:55:36.680Z----------------------------------------------------------------
16092024-01-10T17:55:36.680Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16102024-01-10T17:55:36.680ZStates: Active Active Active
16112024-01-10T17:55:36.680ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16122024-01-10T17:55:36.680Z 1 Acked 1000 Write 9 Done Done Done false
16132024-01-10T17:55:36.680Z 2 Acked 1001 Read 8 Done Done Done false
16142024-01-10T17:55:36.680Z 3 Acked 1002 Read 10 Done Done Done false
16152024-01-10T17:55:36.680Z 4 Acked 1003 Write 2 Sent Done Done false
16162024-01-10T17:55:36.680Z 5 Acked 1004 Read 4 Sent Done Done false
16172024-01-10T17:55:36.680Z 6 Acked 1005 Write 1 Sent Done Done false
16182024-01-10T17:55:36.680Z 7 Acked 1006 Read 5 Sent Done Done false
16192024-01-10T17:55:36.680Z 8 Acked 1007 Read 1 Sent Done Done false
16202024-01-10T17:55:36.680Z 9 Acked 1008 Write 8 Sent Done Done false
16212024-01-10T17:55:36.680Z 10 Acked 1009 Read 3 Sent Done Done false
16222024-01-10T17:55:36.680Z 11 Acked 1010 Read 6 Sent Done Done false
16232024-01-10T17:55:36.680Z 12 Acked 1011 Read 8 Sent Done Done false
16242024-01-10T17:55:36.680Z 13 Acked 1012 Read 2 Sent Done Done false
16252024-01-10T17:55:36.680Z 14 Acked 1013 Read 3 Sent Done Done false
16262024-01-10T17:55:36.680Z 15 Acked 1014 Write 9 Sent Done Done false
16272024-01-10T17:55:36.680Z 16 Acked 1015 Write 10 Sent Done Done false
16282024-01-10T17:55:36.680Z 17 Acked 1016 Write 2 Sent Done Done false
16292024-01-10T17:55:36.680Z 18 Acked 1017 Write 2 Sent Done Done false
16302024-01-10T17:55:36.681Z 19 Acked 1018 Flush 0 Sent Done Done false
16312024-01-10T17:55:36.681Z 20 Acked 1019 Write 7 Sent Done Done false
16322024-01-10T17:55:36.681Z 21 Acked 1020 Write 9 Sent Done Done false
16332024-01-10T17:55:36.681Z 22 Acked 1021 Read 6 Sent Done Done false
16342024-01-10T17:55:36.681Z 23 Acked 1022 Read 3 Sent Done Done false
16352024-01-10T17:55:36.681Z 24 Acked 1023 Write 6 Sent Done Done false
16362024-01-10T17:55:36.681Z 25 Acked 1024 Write 3 Sent Done Done false
16372024-01-10T17:55:36.681Z 26 Acked 1025 Write 6 Sent Done Done false
16382024-01-10T17:55:36.681Z 27 Acked 1026 Read 3 Sent Done Done false
16392024-01-10T17:55:36.681Z 28 Acked 1027 Read 9 Sent Done Done false
16402024-01-10T17:55:36.681Z 29 Acked 1028 Write 6 Sent Done Done false
16412024-01-10T17:55:36.681Z 30 Acked 1029 Read 7 Sent Done Done false
16422024-01-10T17:55:36.682Z STATES DS:0 DS:1 DS:2 TOTAL
16432024-01-10T17:55:36.682Z New 0 0 0 0
16442024-01-10T17:55:36.682Z Sent 27 0 0 27
16452024-01-10T17:55:36.682Z Done 3 30 30 63
16462024-01-10T17:55:36.682Z Skipped 0 0 0 0
16472024-01-10T17:55:36.682Z Error 0 0 0 0
16482024-01-10T17:55:36.682ZLast Flush: 0 1018 1018
16492024-01-10T17:55:36.682ZDownstairs last five completed:
16502024-01-10T17:55:36.682ZUpstairs last five completed: 30 29 28 27 26
16512024-01-10T17:55:36.682ZWrote out file "/tmp/test_repair_verify.data"
16522024-01-10T17:55:36.682Z
16532024-01-10T17:55:36.682ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16542024-01-10T17:55:36.699ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16552024-01-10T17:55:36.699Z 0 000-019 8 14 14 12 21 21 F F F <---
16562024-01-10T17:55:36.700Z 1 020-039 8 8 8 12 12 12 F F F
16572024-01-10T17:55:36.700Z 2 040-059 10 10 10 16 16 16 F F F
16582024-01-10T17:55:36.700Z 3 060-079 12 14 14 19 21 21 F F F <---
16592024-01-10T17:55:36.704Z 4 080-099 12 14 14 19 21 21 F F F <---
16602024-01-10T17:55:36.704Z 5 100-119 13 13 13 20 20 20 F F F
16612024-01-10T17:55:36.704Z 6 120-139 13 13 13 20 20 20 F F F
16622024-01-10T17:55:36.704Z 7 140-159 12 14 14 19 21 21 F F F <---
16632024-01-10T17:55:36.704Z 8 160-179 13 14 14 20 21 21 F F F <---
16642024-01-10T17:55:36.704Z 9 180-199 7 14 14 10 21 21 F T T <---
16652024-01-10T17:55:36.704Z 10 200-219 8 8 8 12 12 12 F F F
16662024-01-10T17:55:36.704Z 11 220-239 12 12 12 19 19 19 F F F
16672024-01-10T17:55:36.704Z 12 240-259 12 14 14 19 21 21 T F F <---
16682024-01-10T17:55:36.704Z 13 260-279 13 13 13 20 20 20 F T T <---
16692024-01-10T17:55:36.704Z 14 280-299 11 11 11 18 18 18 F T T <---
16702024-01-10T17:55:36.704Z 15 300-319 11 11 11 18 18 18 F F F
16712024-01-10T17:55:36.704Z 16 320-339 13 13 13 20 20 20 F F F
16722024-01-10T17:55:36.704Z 17 340-359 12 12 12 19 19 19 F F F
16732024-01-10T17:55:36.704Z 18 360-379 13 13 13 20 20 20 F F F
16742024-01-10T17:55:36.704Z 19 380-399 13 13 13 20 20 20 F F F
16752024-01-10T17:55:36.704Z 20 400-419 13 13 13 20 20 20 F T T <---
16762024-01-10T17:55:36.704Z 21 420-439 12 12 12 19 19 19 F T T <---
16772024-01-10T17:55:36.704Z 22 440-459 13 13 13 20 20 20 F F F
16782024-01-10T17:55:36.704Z 23 460-479 4 14 14 6 21 21 F F F <---
16792024-01-10T17:55:36.704Z 24 480-499 13 13 13 20 20 20 F T T <---
16802024-01-10T17:55:36.704Z 25 500-519 12 12 12 19 19 19 F F F
16812024-01-10T17:55:36.704Z 26 520-539 11 11 11 18 18 18 F F F
16822024-01-10T17:55:36.705Z 27 540-559 9 9 9 13 13 13 F F F
16832024-01-10T17:55:36.705Z 28 560-579 13 13 13 20 20 20 F T T <---
16842024-01-10T17:55:36.705Z 29 580-599 13 13 13 20 20 20 F F F
16852024-01-10T17:55:36.705ZMax gen: 14, Max flush: 21
16862024-01-10T17:55:36.705ZError: Difference in extent metadata found!
16872024-01-10T17:55:36.705ZA Difference in extent metadata is expected here
16882024-01-10T17:55:36.705ZOn loop 7
16892024-01-10T17:55:36.705Z
16902024-01-10T17:55:36.708ZVerifying data now
16912024-01-10T17:55:42.059ZNo change after verify
16922024-01-10T17:55:42.059ZLoop: 7 Downstairs dump after verify (and repair):
16932024-01-10T17:55:42.497ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16942024-01-10T17:55:42.497Z 0 000-019 14 14 14 21 21 21 F F F
16952024-01-10T17:55:42.498Z 1 020-039 8 8 8 12 12 12 F F F
16962024-01-10T17:55:42.498Z 2 040-059 10 10 10 16 16 16 F F F
16972024-01-10T17:55:42.498Z 3 060-079 14 14 14 21 21 21 F F F
16982024-01-10T17:55:42.498Z 4 080-099 14 14 14 21 21 21 F F F
16992024-01-10T17:55:42.498Z 5 100-119 13 13 13 20 20 20 F F F
17002024-01-10T17:55:42.498Z 6 120-139 13 13 13 20 20 20 F F F
17012024-01-10T17:55:42.498Z 7 140-159 14 14 14 21 21 21 F F F
17022024-01-10T17:55:42.498Z 8 160-179 14 14 14 21 21 21 F F F
17032024-01-10T17:55:42.498Z 9 180-199 15 15 15 22 22 22 F F F
17042024-01-10T17:55:42.498Z 10 200-219 8 8 8 12 12 12 F F F
17052024-01-10T17:55:42.499Z 11 220-239 12 12 12 19 19 19 F F F
17062024-01-10T17:55:42.499Z 12 240-259 14 14 14 21 21 21 F F F
17072024-01-10T17:55:42.499Z 13 260-279 15 15 15 22 22 22 F F F
17082024-01-10T17:55:42.499Z 14 280-299 15 15 15 22 22 22 F F F
17092024-01-10T17:55:42.499Z 15 300-319 11 11 11 18 18 18 F F F
17102024-01-10T17:55:42.499Z 16 320-339 13 13 13 20 20 20 F F F
17112024-01-10T17:55:42.499Z 17 340-359 12 12 12 19 19 19 F F F
17122024-01-10T17:55:42.499Z 18 360-379 13 13 13 20 20 20 F F F
17132024-01-10T17:55:42.499Z 19 380-399 13 13 13 20 20 20 F F F
17142024-01-10T17:55:42.499Z 20 400-419 15 15 15 22 22 22 F F F
17152024-01-10T17:55:42.499Z 21 420-439 15 15 15 22 22 22 F F F
17162024-01-10T17:55:42.499Z 22 440-459 13 13 13 20 20 20 F F F
17172024-01-10T17:55:42.499Z 23 460-479 14 14 14 21 21 21 F F F
17182024-01-10T17:55:42.499Z 24 480-499 15 15 15 22 22 22 F F F
17192024-01-10T17:55:42.499Z 25 500-519 12 12 12 19 19 19 F F F
17202024-01-10T17:55:42.499Z 26 520-539 11 11 11 18 18 18 F F F
17212024-01-10T17:55:42.499Z 27 540-559 9 9 9 13 13 13 F F F
17222024-01-10T17:55:42.499Z 28 560-579 15 15 15 22 22 22 F F F
17232024-01-10T17:55:42.499Z 29 580-599 13 13 13 20 20 20 F F F
17242024-01-10T17:55:42.500ZMax gen: 15, Max flush: 22
17252024-01-10T17:55:42.500Z
17262024-01-10T17:55:42.500ZBegin loop 8
17272024-01-10T17:55:42.500ZDownstairs to restart: 2
17282024-01-10T17:55:42.500Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.927701016Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219}
17292024-01-10T17:55:42.500Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.928219441Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219}
17302024-01-10T17:55:42.501Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.928253757Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219}
17312024-01-10T17:55:42.501Z{"msg":"Crucible f936324d-102e-498d-a6d5-b5388dc020a7 has session id: 88b6096f-8dfa-4bba-8548-37c7ebc7fbf1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.928300817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17322024-01-10T17:55:42.501Z{"msg":"Upstairs opts: Upstairs UUID: f936324d-102e-498d-a6d5-b5388dc020a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.928349032Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17332024-01-10T17:55:42.501Z{"msg":"Crucible stats registered with UUID: f936324d-102e-498d-a6d5-b5388dc020a7","v":0,"name":"crucible","level":30Crucible runtime is spawned
17342024-01-10T17:55:42.501Z,"time":"2024-01-10T17:55:41.928432985Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17352024-01-10T17:55:42.501Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.928668647Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219}
17362024-01-10T17:55:42.501Z{"msg":"f936324d-102e-498d-a6d5-b5388dc020a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.928792543Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17372024-01-10T17:55:42.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.928850709Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"0","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17382024-01-10T17:55:42.501Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.92887691Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"0","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17392024-01-10T17:55:42.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.928936426Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"1","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17402024-01-10T17:55:42.502Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.928969548Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"1","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17412024-01-10T17:55:42.502Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.929026809Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"2","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17422024-01-10T17:55:42.502Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.929053196Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"2","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17432024-01-10T17:55:42.502Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.9291829Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17442024-01-10T17:55:42.502Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.929270461Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17452024-01-10T17:55:42.502Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.929319723Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17462024-01-10T17:55:42.502Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.929365812Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17472024-01-10T17:55:42.502Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.929475177Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17482024-01-10T17:55:42.502Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.929553691Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17492024-01-10T17:55:42.502Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.930704508Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"1","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17502024-01-10T17:55:42.503Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.930860716Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"1","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17512024-01-10T17:55:42.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.930927419Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"0","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17522024-01-10T17:55:42.503Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.930953343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"0","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17532024-01-10T17:55:42.503Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.931703557Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"1","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17542024-01-10T17:55:42.504Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.931774134Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"2","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17552024-01-10T17:55:42.504Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.931901014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"2","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17562024-01-10T17:55:42.504Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.932064385Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"0","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17572024-01-10T17:55:42.504Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.932670121Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"1","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17582024-01-10T17:55:42.504Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.932729778Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"1","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17592024-01-10T17:55:42.504Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.932754924Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17602024-01-10T17:55:42.504Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.932775021Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"0","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17612024-01-10T17:55:42.504Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.932796614Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"0","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17622024-01-10T17:55:42.504Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.932856213Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17632024-01-10T17:55:42.504Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.933218827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"2","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17642024-01-10T17:55:42.504Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.933927142Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"2","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17652024-01-10T17:55:42.504Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.933999068Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"2","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17662024-01-10T17:55:42.504Z{"msg":"[0]R flush_numbers[0..12]: [21, 12, 16, 21, 21, 20, 20, 21, 21, 22, 12, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934036483Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17672024-01-10T17:55:42.504Z{"msg":"[0]R generation[0..12]: [14, 8, 10, 14, 14, 13, 13, 14, 14, 15, 8, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934081716Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17682024-01-10T17:55:42.504Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934107581Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17692024-01-10T17:55:42.504Z{"msg":"[1]R flush_numbers[0..12]: [21, 12, 16, 21, 21, 20, 20, 21, 21, 22, 12, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934130573Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17702024-01-10T17:55:42.505Z{"msg":"[1]R generation[0..12]: [14, 8, 10, 14, 14, 13, 13, 14, 14, 15, 8, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.93417049Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17712024-01-10T17:55:42.505Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934195414Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17722024-01-10T17:55:42.505Z{"msg":"[2]R flush_numbers[0..12]: [21, 12, 16, 21, 21, 20, 20, 21, 21, 22, 12, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934218442Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"Wait for a query_work_queue command to finish before sending IO
17732024-01-10T17:55:42.506Z}
17742024-01-10T17:55:42.506Z{"msg":"[2]R generation[0..12]: [14, 8, 10, 14, 14, 13, 13, 14, 14, 15, 8, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934300939Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17752024-01-10T17:55:42.506Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934337604Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17762024-01-10T17:55:42.506Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934361124Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17772024-01-10T17:55:42.506Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934416893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17782024-01-10T17:55:42.506Z{"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934452977Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17792024-01-10T17:55:42.506Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934476891Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"":"downstairs","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17802024-01-10T17:55:42.507Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934519846Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17812024-01-10T17:55:42.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934541602Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"0","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17822024-01-10T17:55:42.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934563913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"1","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17832024-01-10T17:55:42.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934625101Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"client":"2","":"downstairs","session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17842024-01-10T17:55:42.507Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934652185Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17852024-01-10T17:55:42.507Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934672275Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17862024-01-10T17:55:42.507Z{"msg":"f936324d-102e-498d-a6d5-b5388dc020a7 is now active with session: 88b6096f-8dfa-4bba-8548-37c7ebc7fbf1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934732269Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17872024-01-10T17:55:42.507Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.934758058Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219,"session_id":"88b6096f-8dfa-4bba-8548-37c7ebc7fbf1"}
17882024-01-10T17:55:42.507Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:41.93477836Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1219}
17892024-01-10T17:55:42.507ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
17902024-01-10T17:55:42.507ZLoading write count information from file "/tmp/test_repair_verify.data"
17912024-01-10T17:55:42.507ZRun Repair workload
17922024-01-10T17:55:42.508Z01/30 Flush
17932024-01-10T17:55:42.746Z02/30 Write block 65 len 8192 data: 2 3
17942024-01-10T17:55:42.746Z03/30 Read block 469 len 12288
17952024-01-10T17:55:42.746Z04/30 Flush
17962024-01-10T17:55:42.746Z05/30 Write block 443 len 16384 data: 5 6 7 7
17972024-01-10T17:55:42.746Z06/30 Write block 319 len 32768 data: 2 2 2 3 2 4 4 3
17982024-01-10T17:55:42.746Z07/30 Read block 484 len 12288
17992024-01-10T17:55:42.746Z08/30 Flush
18002024-01-10T17:55:42.746Z09/30 Write block 49 len 32768 data: 3 3 3 3 3 4 4 3
18012024-01-10T17:55:42.746Z10/30 Write block 552 len 24576 data: 3 2 2 2 2 2
18022024-01-10T17:55:42.746Z11/30 Write block 545 len 4096 data: 4
18032024-01-10T17:55:42.746Z12/30 Read block 420 len 36864
18042024-01-10T17:55:42.747Z13/30 Read block 11 len 16384
18052024-01-10T17:55:42.747Z14/30 Read block 448 len 20480
18062024-01-10T17:55:42.747Z15/30 Read block 20 len 36864
18072024-01-10T17:55:42.747Z16/30 Flush
18082024-01-10T17:55:42.747Z17/30 Write block 443 len 32768 data: 6 7 8 8 5 4 3 3
18092024-01-10T17:55:42.747Z18/30 Write block 255 len 28672 data: 2 2 2 3 2 2 2
18102024-01-10T17:55:42.747Z19/30 Read block 338 len 24576
18112024-01-10T17:55:42.747Z20/30 Read block 465 len 20480
18122024-01-10T17:55:42.747Z21/30 Read block 69 len 8192
18132024-01-10T17:55:42.747Z22/30 Read block 487 len 20480
18142024-01-10T17:55:42.747Z23/30 Flush
18152024-01-10T17:55:42.747Z24/30 Write block 382 len 4096 data: 3
18162024-01-10T17:55:42.747Z25/30 Write block 40 len 12288 data: 2 2 2
18172024-01-10T17:55:42.763Z26/30 Write block 161 len 12288 data: 3 6 6
18182024-01-10T17:55:42.763Z27/30 Read block 557 len 32768
18192024-01-10T17:55:42.763Z28/30 Read block 312 len 4096
18202024-01-10T17:55:42.763Z29/30 Write block 41 len 20480 data: 3 3 2 3 3
18212024-01-10T17:55:42.764Z30/30 Read block 591 len 36864
18222024-01-10T17:55:42.764Z----------------------------------------------------------------
18232024-01-10T17:55:42.764Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:29
18242024-01-10T17:55:42.764ZStates: Active Active Active
18252024-01-10T17:55:42.764ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18262024-01-10T17:55:42.764Z 2 Acked 1001 Write 2 Done Done Done false
18272024-01-10T17:55:42.764Z 3 Acked 1002 Read 3 Done Done Done false
18282024-01-10T17:55:42.764Z 4 Acked 1003 Flush 0 Done Done Sent false
18292024-01-10T17:55:42.764Z 5 Acked 1004 Write 4 Done Done Sent false
18302024-01-10T17:55:42.764Z 6 Acked 1005 Write 8 Done Done Sent false
18312024-01-10T17:55:42.764Z 7 Acked 1006 Read 3 Done Done Sent false
18322024-01-10T17:55:42.764Z 8 Acked 1007 Flush 0 Done Done Sent false
18332024-01-10T17:55:42.764Z 9 Acked 1008 Write 8 Done Done Sent false
18342024-01-10T17:55:42.764Z 10 Acked 1009 Write 6 Done Done Sent false
18352024-01-10T17:55:42.764Z 11 Acked 1010 Write 1 Done Done Sent false
18362024-01-10T17:55:42.764Z 12 Acked 1011 Read 9 Done Done Sent false
18372024-01-10T17:55:42.764Z 13 Acked 1012 Read 4 Done Done Sent false
18382024-01-10T17:55:42.764Z 14 Acked 1013 Read 5 Done Done Sent false
18392024-01-10T17:55:42.765Z 15 Acked 1014 Read 9 Done Done Sent false
18402024-01-10T17:55:42.765Z 16 Acked 1015 Flush 0 Done Done Sent false
18412024-01-10T17:55:42.768Z 17 Acked 1016 Write 8 Done Done Sent false
18422024-01-10T17:55:42.768Z 18 Acked 1017 Write 7 Done Done Sent false
18432024-01-10T17:55:42.768Z 19 Acked 1018 Read 6 Done Done Sent false
18442024-01-10T17:55:42.768Z 20 Acked 1019 Read 5 Done Done Sent false
18452024-01-10T17:55:42.768Z 21 Acked 1020 Read 2 Done Done Sent false
18462024-01-10T17:55:42.768Z 22 Acked 1021 Read 5 Done Done Sent false
18472024-01-10T17:55:42.768Z 23 Acked 1022 Flush 0 Done Done Sent false
18482024-01-10T17:55:42.768Z 24 Acked 1023 Write 1 Done Done Sent false
18492024-01-10T17:55:42.769Z 25 Acked 1024 Write 3 Done Done Sent false
18502024-01-10T17:55:42.769Z 26 Acked 1025 Write 3 Done Done Sent false
18512024-01-10T17:55:42.769Z 27 Acked 1026 Read 8 Done Done Sent false
18522024-01-10T17:55:42.769Z 28 Acked 1027 Read 1 Done Done Sent false
18532024-01-10T17:55:42.769Z 29 Acked 1028 Write 5 Done Done Sent false
18542024-01-10T17:55:42.769Z 30 Acked 1029 Read 9 Done Sent Sent false
18552024-01-10T17:55:42.769Z STATES DS:0 DS:1 DS:2 TOTAL
18562024-01-10T17:55:42.769Z New 0 0 0 0
18572024-01-10T17:55:42.769Z Sent 0 1 27 28
18582024-01-10T17:55:42.769Z Done 29 28 2 59
18592024-01-10T17:55:42.769Z Skipped 0 0 0 0
18602024-01-10T17:55:42.769Z Error 0 0 0 0
18612024-01-10T17:55:42.769ZLast Flush: 1022 1022 1000
18622024-01-10T17:55:42.769ZDownstairs last five completed: 1000
18632024-01-10T17:55:42.769ZUpstairs last five completed: 30 29 28 27 26
18642024-01-10T17:55:42.769ZWrote out file "/tmp/test_repair_verify.data"
18652024-01-10T17:55:42.773Z
18662024-01-10T17:55:42.773ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18672024-01-10T17:55:42.773ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18682024-01-10T17:55:42.773Z 0 000-019 14 14 14 21 21 21 F F F
18692024-01-10T17:55:42.773Z 1 020-039 8 8 8 12 12 12 F F F
18702024-01-10T17:55:42.773Z 2 040-059 16 16 10 26 26 16 T T F <---
18712024-01-10T17:55:42.773Z 3 060-079 16 16 14 24 24 21 F F T <---
18722024-01-10T17:55:42.774Z 4 080-099 14 14 14 21 21 21 F F F
18732024-01-10T17:55:42.774Z 5 100-119 13 13 13 20 20 20 F F F
18742024-01-10T17:55:42.774Z 6 120-139 13 13 13 20 20 20 F F F
18752024-01-10T17:55:42.774Z 7 140-159 14 14 14 21 21 21 F F F
18762024-01-10T17:55:42.774Z 8 160-179 14 14 14 21 21 21 T T F <---
18772024-01-10T17:55:42.774Z 9 180-199 15 15 15 22 22 22 F F F
18782024-01-10T17:55:42.774Z 10 200-219 8 8 8 12 12 12 F F F
18792024-01-10T17:55:42.774Z 11 220-239 12 12 12 19 19 19 F F F
18802024-01-10T17:55:42.774Z 12 240-259 16 16 14 27 27 21 F F F <---
18812024-01-10T17:55:42.774Z 13 260-279 16 16 15 27 27 22 F F F <---
18822024-01-10T17:55:42.774Z 14 280-299 15 15 15 22 22 22 F F F
18832024-01-10T17:55:42.774Z 15 300-319 16 16 11 25 25 18 F F F <---
18842024-01-10T17:55:42.778Z 16 320-339 16 16 13 25 25 20 F F F <---
18852024-01-10T17:55:42.778Z 17 340-359 12 12 12 19 19 19 F F F
18862024-01-10T17:55:42.778Z 18 360-379 13 13 13 20 20 20 F F F
18872024-01-10T17:55:42.778Z 19 380-399 13 13 13 20 20 20 T T F <---
18882024-01-10T17:55:42.778Z 20 400-419 15 15 15 22 22 22 F F F
18892024-01-10T17:55:42.779Z 21 420-439 15 15 15 22 22 22 F F F
18902024-01-10T17:55:42.780Z 22 440-459 16 16 13 27 27 20 F F F <---
18912024-01-10T17:55:42.780Z 23 460-479 14 14 14 21 21 21 F F F
18922024-01-10T17:55:42.780Z 24 480-499 15 15 15 22 22 22 F F F
18932024-01-10T17:55:42.871Z 25 500-519 12 12 12 19 19 19 F F F
18942024-01-10T17:55:42.871Z 26 520-539 11 11 11 18 18 18 F F F
18952024-01-10T17:55:42.872Z 27 540-559 16 16 9 26 26 13 F F F <---
18962024-01-10T17:55:42.872Z 28 560-579 15 15 15 22 22 22 F F F
18972024-01-10T17:55:42.872Z 29 580-599 13 13 13 20 20 20 F F F
18982024-01-10T17:55:42.872ZMax gen: 16, Max flush: 27
18992024-01-10T17:55:42.872ZError: Difference in extent metadata found!
19002024-01-10T17:55:42.872ZA Difference in extent metadata is expected here
19012024-01-10T17:55:42.872ZOn loop 8
19022024-01-10T17:55:42.872Z
19032024-01-10T17:55:42.872ZVerifying data now
19042024-01-10T17:55:43.331ZNo change after verify
19052024-01-10T17:55:43.331ZLoop: 8 Downstairs dump after verify (and repair):
19062024-01-10T17:55:43.445ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19072024-01-10T17:55:43.445Z 0 000-019 14 14 14 21 21 21 F F F
19082024-01-10T17:55:43.446Z 1 020-039 8 8 8 12 12 12 F F F
19092024-01-10T17:55:43.446Z 2 040-059 17 17 17 28 28 28 F F F
19102024-01-10T17:55:43.446Z 3 060-079 16 16 16 24 24 24 F F F
19112024-01-10T17:55:43.446Z 4 080-099 14 14 14 21 21 21 F F F
19122024-01-10T17:55:43.446Z 5 100-119 13 13 13 20 20 20 F F F
19132024-01-10T17:55:43.481Z 6 120-139 13 13 13 20 20 20 F F F
19142024-01-10T17:55:43.481Z 7 140-159 14 14 14 21 21 21 F F F
19152024-01-10T17:55:43.481Z 8 160-179 17 17 17 28 28 28 F F F
19162024-01-10T17:55:43.481Z 9 180-199 15 15 15 22 22 22 F F F
19172024-01-10T17:55:43.482Z 10 200-219 8 8 8 12 12 12 F F F
19182024-01-10T17:55:43.482Z 11 220-239 12 12 12 19 19 19 F F F
19192024-01-10T17:55:43.482Z 12 240-259 16 16 16 27 27 27 F F F
19202024-01-10T17:55:43.482Z 13 260-279 16 16 16 27 27 27 F F F
19212024-01-10T17:55:43.482Z 14 280-299 15 15 15 22 22 22 F F F
19222024-01-10T17:55:43.482Z 15 300-319 16 16 16 25 25 25 F F F
19232024-01-10T17:55:43.482Z 16 320-339 16 16 16 25 25 25 F F F
19242024-01-10T17:55:43.482Z 17 340-359 12 12 12 19 19 19 F F F
19252024-01-10T17:55:43.482Z 18 360-379 13 13 13 20 20 20 F F F
19262024-01-10T17:55:43.482Z 19 380-399 17 17 17 28 28 28 F F F
19272024-01-10T17:55:43.482Z 20 400-419 15 15 15 22 22 22 F F F
19282024-01-10T17:55:43.482Z 21 420-439 15 15 15 22 22 22 F F F
19292024-01-10T17:55:43.482Z 22 440-459 16 16 16 27 27 27 F F F
19302024-01-10T17:55:43.482Z 23 460-479 14 14 14 21 21 21 F F F
19312024-01-10T17:55:43.482Z 24 480-499 15 15 15 22 22 22 F F F
19322024-01-10T17:55:43.482Z 25 500-519 12 12 12 19 19 19 F F F
19332024-01-10T17:55:43.482Z 26 520-539 11 11 11 18 18 18 F F F
19342024-01-10T17:55:43.483Z 27 540-559 16 16 16 26 26 26 F F F
19352024-01-10T17:55:43.483Z 28 560-579 15 15 15 22 22 22 F F F
19362024-01-10T17:55:43.483Z 29 580-599 13 13 13 20 20 20 F F F
19372024-01-10T17:55:43.483ZMax gen: 17, Max flush: 28
19382024-01-10T17:55:43.507Z
19392024-01-10T17:55:43.507ZBegin loop 9
19402024-01-10T17:55:43.508ZDownstairs to restart: 1
19412024-01-10T17:55:43.508Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.13231467Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227}
19422024-01-10T17:55:43.508Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.13281902Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227}
19432024-01-10T17:55:43.508Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"Crucible runtime is spawned
19442024-01-10T17:55:43.508Zcrucible","level":30,"time":"2024-01-10T17:55:43.132982402Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227}
19452024-01-10T17:55:43.508Z{"msg":"Crucible 4d7f989e-1afb-4f62-a1ac-61db48dbc887 has session id: 379063de-09b9-4b48-8992-0ab9f8aa0945","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133028848Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19462024-01-10T17:55:43.508Z{"msg":"Upstairs opts: Upstairs UUID: 4d7f989e-1afb-4f62-a1ac-61db48dbc887, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.13309862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19472024-01-10T17:55:43.508Z{"msg":"Crucible stats registered with UUID: 4d7f989e-1afb-4f62-a1ac-61db48dbc887","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133125901Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19482024-01-10T17:55:43.508Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133183265Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227}
19492024-01-10T17:55:43.508Z{"msg":"4d7f989e-1afb-4f62-a1ac-61db48dbc887 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133406306Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19502024-01-10T17:55:43.508Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.13346479Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"0","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19512024-01-10T17:55:43.508Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133490244Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"0","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19522024-01-10T17:55:43.508Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133517931Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"1","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19532024-01-10T17:55:43.508Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133590626Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"1","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19542024-01-10T17:55:43.508Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133631376Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"2","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19552024-01-10T17:55:43.508Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.13369513Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"2","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19562024-01-10T17:55:43.508Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133723692Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19572024-01-10T17:55:43.509Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133747071Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19582024-01-10T17:55:43.509Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.133967406Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19592024-01-10T17:55:43.509Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.134051407Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19602024-01-10T17:55:43.509Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.134100432Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19612024-01-10T17:55:43.530Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.134142603Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19622024-01-10T17:55:43.531Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.135187938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"0","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19632024-01-10T17:55:43.531Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.135290781Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"0","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19642024-01-10T17:55:43.531Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.135337044Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"2","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19652024-01-10T17:55:43.531Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.135430974Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"2","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19662024-01-10T17:55:43.531Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.135749683Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"1","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19672024-01-10T17:55:43.531Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.135811157Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"1","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19682024-01-10T17:55:43.531Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.136663973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"2","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19692024-01-10T17:55:43.531Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.136735624Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"0","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19702024-01-10T17:55:43.531Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.136857174Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"1","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19712024-01-10T17:55:43.551Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.137644459Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"2","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19722024-01-10T17:55:43.551Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.137723872Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"2","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19732024-01-10T17:55:43.551Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.13774981Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19742024-01-10T17:55:43.551Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.13777166Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"0","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19752024-01-10T17:55:43.552Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.137858997Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"0","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19762024-01-10T17:55:43.552Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.137891248Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19772024-01-10T17:55:43.560Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.137932275Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"1","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19782024-01-10T17:55:43.561Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.137956381Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"1","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19792024-01-10T17:55:43.561Z{"msg":"[0]R flush_numbers[0..12]: [21, 12, 28, 24, 21, 20, 20, 21, 28, 22, 12, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.137980319Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19802024-01-10T17:55:43.561Z{"msg":"[0]R generation[0..12]: [14, 8, 17, 16, 14, 13, 13, 14, 17, 15, 8, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138021512Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19812024-01-10T17:55:43.567Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138047017Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19822024-01-10T17:55:43.567Z{"msg":"[1]R flush_numbers[0..12]: [21, 12, 28, 24, 21, 20, 20, 21, 28, 22, 12, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138096324Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19832024-01-10T17:55:43.567Z{"msg":"Wait for a query_work_queue command to finish before sending IO
19842024-01-10T17:55:43.568Z[1]R generation[0..12]: [14, 8, 17, 16, 14, 13, 13, 14, 17, 15, 8, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138123145Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19852024-01-10T17:55:43.568Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138167017Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19862024-01-10T17:55:43.568Z{"msg":"[2]R flush_numbers[0..12]: [21, 12, 28, 24, 21, 20, 20, 21, 28, 22, 12, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138191308Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19872024-01-10T17:55:43.568Z{"msg":"[2]R generation[0..12]: [14, 8, 17, 16, 14, 13, 13, 14, 17, 15, 8, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138232593Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19882024-01-10T17:55:43.568Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138257561Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19892024-01-10T17:55:43.568Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138301433Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19902024-01-10T17:55:43.568Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138324801Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19912024-01-10T17:55:43.574Z{"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138347212Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19922024-01-10T17:55:43.574Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138368798Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"":"downstairs","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19932024-01-10T17:55:43.574Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138390461Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19942024-01-10T17:55:43.574Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138411398Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"0","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19952024-01-10T17:55:43.575Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138442533Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"1","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19962024-01-10T17:55:43.575Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138484682Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"client":"2","":"downstairs","session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19972024-01-10T17:55:43.575Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138544123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19982024-01-10T17:55:43.575Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138568925Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
19992024-01-10T17:55:43.575Z{"msg":"4d7f989e-1afb-4f62-a1ac-61db48dbc887 is now active with session: 379063de-09b9-4b48-8992-0ab9f8aa0945","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.13859056Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
20002024-01-10T17:55:43.575Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138651469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227,"session_id":"379063de-09b9-4b48-8992-0ab9f8aa0945"}
20012024-01-10T17:55:43.575Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:43.138694401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1227}
20022024-01-10T17:55:43.598ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20032024-01-10T17:55:43.598ZLoading write count information from file "/tmp/test_repair_verify.data"
20042024-01-10T17:55:43.598ZRun Repair workload
20052024-01-10T17:55:43.598Z01/30 Write block 197 len 24576 data: 3 3 4 3 3 3
20062024-01-10T17:55:43.786Z02/30 Write block 495 len 12288 data: 5 5 5
20072024-01-10T17:55:43.786Z03/30 Flush
20082024-01-10T17:55:43.786Z04/30 Write block 511 len 28672 data: 4 5 6 5 4 5 5
20092024-01-10T17:55:43.786Z05/30 Write block 281 len 20480 data: 2 2 2 2 2
20102024-01-10T17:55:43.786Z06/30 Read block 338 len 4096
20112024-01-10T17:55:43.786Z07/30 Write block 270 len 32768 data: 4 4 3 3 4 4 4 3
20122024-01-10T17:55:43.787Z08/30 Flush
20132024-01-10T17:55:43.787Z09/30 Flush
20142024-01-10T17:55:43.787Z10/30 Write block 291 len 12288 data: 3 2 2
20152024-01-10T17:55:43.787Z11/30 Read block 251 len 36864
20162024-01-10T17:55:43.787Z12/30 Read block 451 len 24576
20172024-01-10T17:55:43.787Z13/30 Write block 64 len 24576 data: 2 3 4 2 2 2
20182024-01-10T17:55:43.787Z14/30 Write block 568 len 40960 data: 3 3 3 3 2 2 2 2 3 3
20192024-01-10T17:55:43.787Z15/30 Read block 391 len 20480
20202024-01-10T17:55:43.787Z16/30 Read block 430 len 32768
20212024-01-10T17:55:43.787Z17/30 Read block 231 len 12288
20222024-01-10T17:55:43.787Z18/30 Write block 436 len 32768 data: 2 2 2 4 4 5 5 7
20232024-01-10T17:55:43.787Z19/30 Read block 525 len 20480
20242024-01-10T17:55:43.787Z20/30 Read block 40 len 20480
20252024-01-10T17:55:43.787Z21/30 Read block 232 len 8192
20262024-01-10T17:55:43.788Z22/30 Write block 492 len 12288 data: 3 3 4
20272024-01-10T17:55:43.788Z23/30 Write block 81 len 16384 data: 3 2 2 2
20282024-01-10T17:55:43.788Z24/30 Flush
20292024-01-10T17:55:43.788Z25/30 Write block 466 len 20480 data: 4 5 5 3 3
20302024-01-10T17:55:43.788Z26/30 Write block 387 len 4096 data: 2
20312024-01-10T17:55:43.788Z27/30 Write block 0 len 28672 data: 2 2 2 2 2 2 2
20322024-01-10T17:55:43.788Z28/30 Write block 428 len 28672 data: 2 2 2 2 2 2 2
20332024-01-10T17:55:43.788Z29/30 Write block 188 len 24576 data: 3 3 3 2 2 2
20342024-01-10T17:55:43.788Z30/30 Write block 275 len 24576 data: 5 5 4 2 2 2
20352024-01-10T17:55:43.788Z----------------------------------------------------------------
20362024-01-10T17:55:43.788Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:27
20372024-01-10T17:55:43.789ZStates: Active Active Active
20382024-01-10T17:55:43.789ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20392024-01-10T17:55:43.789Z 4 Acked 1003 Write 7 Done Sent Done false
20402024-01-10T17:55:43.789Z 5 Acked 1004 Write 5 Done Sent Done false
20412024-01-10T17:55:43.789Z 6 Acked 1005 Read 1 Done Sent Done false
20422024-01-10T17:55:43.789Z 7 Acked 1006 Write 8 Done Sent Done false
20432024-01-10T17:55:43.789Z 8 Acked 1007 Flush 0 Done Sent Done false
20442024-01-10T17:55:43.789Z 9 Acked 1008 Flush 0 Done Sent Done false
20452024-01-10T17:55:43.789Z 10 Acked 1009 Write 3 Done Sent Done false
20462024-01-10T17:55:43.789Z 11 Acked 1010 Read 9 Done Sent Done false
20472024-01-10T17:55:43.789Z 12 Acked 1011 Read 6 Done Sent Done false
20482024-01-10T17:55:43.789Z 13 Acked 1012 Write 6 Done Sent Done false
20492024-01-10T17:55:43.789Z 14 Acked 1013 Write 10 Done Sent Done false
20502024-01-10T17:55:43.790Z 15 Acked 1014 Read 5 Done Sent Done false
20512024-01-10T17:55:43.790Z 16 Acked 1015 Read 8 Done Sent Done false
20522024-01-10T17:55:43.790Z 17 Acked 1016 Read 3 Done Sent Done false
20532024-01-10T17:55:43.790Z 18 Acked 1017 Write 8 Done Sent Done false
20542024-01-10T17:55:43.790Z 19 Acked 1018 Read 5 Done Sent Done false
20552024-01-10T17:55:43.790Z 20 Acked 1019 Read 5 Done Sent Done false
20562024-01-10T17:55:43.790Z 21 Acked 1020 Read 2 Done Sent Done false
20572024-01-10T17:55:43.790Z 22 Acked 1021 Write 3 Done Sent Done false
20582024-01-10T17:55:43.790Z 23 Acked 1022 Write 4 Done Sent Done false
20592024-01-10T17:55:43.790Z 24 Acked 1023 Flush 0 Done Sent Done false
20602024-01-10T17:55:43.790Z 25 Acked 1024 Write 5 Done Sent Done false
20612024-01-10T17:55:43.790Z 26 Acked 1025 Write 1 Done Sent Done false
20622024-01-10T17:55:43.790Z 27 Acked 1026 Write 7 Done Sent Done false
20632024-01-10T17:55:43.790Z 28 Acked 1027 Write 7 Done Sent Done false
20642024-01-10T17:55:43.791Z 29 Acked 1028 Write 6 Done Sent Done false
20652024-01-10T17:55:43.791Z 30 Acked 1029 Write 6 Sent Sent Sent false
20662024-01-10T17:55:43.791Z STATES DS:0 DS:1 DS:2 TOTAL
20672024-01-10T17:55:43.791Z New 0 0 0 0
20682024-01-10T17:55:43.791Z Sent 1 27 1 29
20692024-01-10T17:55:43.791Z Done 26 0 26 52
20702024-01-10T17:55:43.833Z Skipped 0 0 0 0
20712024-01-10T17:55:43.833Z Error 0 0 0 0
20722024-01-10T17:55:43.833ZLast Flush: 1023 1002 1023
20732024-01-10T17:55:43.833ZDownstairs last five completed: 1002 1001 1000
20742024-01-10T17:55:43.833ZUpstairs last five completed: 30 29 28 27 26
20752024-01-10T17:55:43.833ZWrote out file "/tmp/test_repair_verify.data"
20762024-01-10T17:55:43.833Z
20772024-01-10T17:55:43.834ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20782024-01-10T17:55:43.834ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20792024-01-10T17:55:43.834Z 0 000-019 14 14 14 21 21 21 T F T <---
20802024-01-10T17:55:43.834Z 1 020-039 8 8 8 12 12 12 F F F
20812024-01-10T17:55:43.834Z 2 040-059 17 17 17 28 28 28 F F F
20822024-01-10T17:55:43.834Z 3 060-079 18 16 18 32 24 32 F F F <---
20832024-01-10T17:55:43.834Z 4 080-099 18 14 18 32 21 32 F F F <---
20842024-01-10T17:55:43.834Z 5 100-119 13 13 13 20 20 20 F F F
20852024-01-10T17:55:43.834Z 6 120-139 13 13 13 20 20 20 F F F
20862024-01-10T17:55:43.834Z 7 140-159 14 14 14 21 21 21 F F F
20872024-01-10T17:55:43.834Z 8 160-179 17 17 17 28 28 28 F F F
20882024-01-10T17:55:43.834Z 9 180-199 18 18 18 29 29 29 T F T <---
20892024-01-10T17:55:43.834Z 10 200-219 18 18 18 29 29 29 F F F
20902024-01-10T17:55:43.834Z 11 220-239 12 12 12 19 19 19 F F F
20912024-01-10T17:55:43.835Z 12 240-259 16 16 16 27 27 27 F F F
20922024-01-10T17:55:43.835Z 13 260-279 18 16 18 30 27 30 T F T <---
20932024-01-10T17:55:43.835Z 14 280-299 18 15 18 32 22 32 T F T <---
20942024-01-10T17:55:43.835Z 15 300-319 16 16 16 25 25 25 F F F
20952024-01-10T17:55:43.835Z 16 320-339 16 16 16 25 25 25 F F F
20962024-01-10T17:55:43.835Z 17 340-359 12 12 12 19 19 19 F F F
20972024-01-10T17:55:43.835Z 18 360-379 13 13 13 20 20 20 F F F
20982024-01-10T17:55:43.835Z 19 380-399 17 17 17 28 28 28 T F T <---
20992024-01-10T17:55:43.835Z 20 400-419 15 15 15 22 22 22 F F F
21002024-01-10T17:55:43.835Z 21 420-439 18 15 18 32 22 32 T F T <---
21012024-01-10T17:55:43.835Z 22 440-459 18 16 18 32 27 32 F F F <---
21022024-01-10T17:55:43.835Z 23 460-479 14 14 14 21 21 21 T F T <---
21032024-01-10T17:55:43.835Z 24 480-499 18 18 18 32 29 32 F F F <---
21042024-01-10T17:55:43.836Z 25 500-519 18 12 18 30 19 30 F F F <---
21052024-01-10T17:55:43.836Z 26 520-539 11 11 11 18 18 18 F F F
21062024-01-10T17:55:44.343Z 27 540-559 16 16 16 26 26 26 F F F
21072024-01-10T17:55:44.343Z 28 560-579 18 15 18 32 22 32 F F F <---
21082024-01-10T17:55:44.393Z 29 580-599 13 13 13 20 20 20 F F F
21092024-01-10T17:55:44.393ZMax gen: 18, Max flush: 32
21102024-01-10T17:55:44.393ZError: Difference in extent metadata found!
21112024-01-10T17:55:44.393ZA Difference in extent metadata is expected here
21122024-01-10T17:55:44.393ZOn loop 9
21132024-01-10T17:55:44.393Z
21142024-01-10T17:55:44.393ZVerifying data now
21152024-01-10T17:56:07.814ZNo change after verify
21162024-01-10T17:56:07.814ZLoop: 9 Downstairs dump after verify (and repair):
21172024-01-10T17:56:08.481ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21182024-01-10T17:56:08.481Z 0 000-019 19 19 19 33 33 33 F F F
21192024-01-10T17:56:08.482Z 1 020-039 8 8 8 12 12 12 F F F
21202024-01-10T17:56:08.482Z 2 040-059 17 17 17 28 28 28 F F F
21212024-01-10T17:56:08.482Z 3 060-079 18 18 18 32 32 32 F F F
21222024-01-10T17:56:08.482Z 4 080-099 18 18 18 32 32 32 F F F
21232024-01-10T17:56:08.482Z 5 100-119 13 13 13 20 20 20 F F F
21242024-01-10T17:56:08.482Z 6 120-139 13 13 13 20 20 20 F F F
21252024-01-10T17:56:08.482Z 7 140-159 14 14 14 21 21 21 F F F
21262024-01-10T17:56:08.482Z 8 160-179 17 17 17 28 28 28 F F F
21272024-01-10T17:56:08.483Z 9 180-199 19 19 19 33 33 33 F F F
21282024-01-10T17:56:08.483Z 10 200-219 18 18 18 29 29 29 F F F
21292024-01-10T17:56:08.483Z 11 220-239 12 12 12 19 19 19 F F F
21302024-01-10T17:56:08.483Z 12 240-259 16 16 16 27 27 27 F F F
21312024-01-10T17:56:08.483Z 13 260-279 19 19 19 33 33 33 F F F
21322024-01-10T17:56:08.483Z 14 280-299 19 19 19 33 33 33 F F F
21332024-01-10T17:56:08.483Z 15 300-319 16 16 16 25 25 25 F F F
21342024-01-10T17:56:08.483Z 16 320-339 16 16 16 25 25 25 F F F
21352024-01-10T17:56:08.483Z 17 340-359 12 12 12 19 19 19 F F F
21362024-01-10T17:56:08.483Z 18 360-379 13 13 13 20 20 20 F F F
21372024-01-10T17:56:08.483Z 19 380-399 19 19 19 33 33 33 F F F
21382024-01-10T17:56:08.483Z 20 400-419 15 15 15 22 22 22 F F F
21392024-01-10T17:56:08.483Z 21 420-439 19 19 19 33 33 33 F F F
21402024-01-10T17:56:08.483Z 22 440-459 18 18 18 32 32 32 F F F
21412024-01-10T17:56:08.483Z 23 460-479 19 19 19 33 33 33 F F F
21422024-01-10T17:56:08.483Z 24 480-499 18 18 18 32 32 32 F F F
21432024-01-10T17:56:08.483Z 25 500-519 18 18 18 30 30 30 F F F
21442024-01-10T17:56:08.483Z 26 520-539 11 11 11 18 18 18 F F F
21452024-01-10T17:56:08.483Z 27 540-559 16 16 16 26 26 26 F F F
21462024-01-10T17:56:08.483Z 28 560-579 18 18 18 32 32 32 F F F
21472024-01-10T17:56:08.483Z 29 580-599 13 13 13 20 20 20 F F F
21482024-01-10T17:56:08.483ZMax gen: 19, Max flush: 33
21492024-01-10T17:56:08.484Z
21502024-01-10T17:56:08.484ZBegin loop 10
21512024-01-10T17:56:08.484ZDownstairs to restart: 0
21522024-01-10T17:56:08.484Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.577522144Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235}
21532024-01-10T17:56:08.486Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.578046397Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235}
21542024-01-10T17:56:08.486Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.578078494Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235}
21552024-01-10T17:56:08.486Z{"msg":"Crucible 5d8b9143-5cc3-4fb3-a0dd-cab5e217d6d7 has session id: 0f3aa5af-9da1-4a31-8ecb-45599946a7f3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.57812016Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21562024-01-10T17:56:08.486Z{"msg":"Upstairs opts: Upstairs UUID: 5d8b9143-5cc3-4fb3-a0dd-cab5e217d6d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.578164363Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21572024-01-10T17:56:08.486Z{"msg":"Crucible stats registered with UUID: 5d8b9143-5cc3-4fb3-a0dd-cab5e217d6d7","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.578222943Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21582024-01-10T17:56:08.486ZCrucible runtime is spawned
21592024-01-10T17:56:08.486Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.578538526Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235}
21602024-01-10T17:56:08.487Z{"msg":"5d8b9143-5cc3-4fb3-a0dd-cab5e217d6d7 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.578724604Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21612024-01-10T17:56:08.487Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.57878569Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"0","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21622024-01-10T17:56:08.487Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.578853788Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"0","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21632024-01-10T17:56:08.488Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.57892681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"1","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21642024-01-10T17:56:08.488Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.57897239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"1","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21652024-01-10T17:56:08.488Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.579100947Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"2","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21662024-01-10T17:56:08.488Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.579340441Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"2","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21672024-01-10T17:56:08.488Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.579382753Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21682024-01-10T17:56:08.488Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.579419919Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21692024-01-10T17:56:08.488Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.579461951Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21702024-01-10T17:56:08.488Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.579552681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21712024-01-10T17:56:08.488Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.579596912Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21722024-01-10T17:56:08.488Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.579625245Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21732024-01-10T17:56:08.488Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.580433231Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"1","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21742024-01-10T17:56:08.488Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.580494976Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"1","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21752024-01-10T17:56:08.488Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.580529462Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"2","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21762024-01-10T17:56:08.489Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.580683996Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"2","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21772024-01-10T17:56:08.489Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.581505215Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"2","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21782024-01-10T17:56:08.489Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.581721956Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"1","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21792024-01-10T17:56:08.489Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.58176991Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"0","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21802024-01-10T17:56:08.489Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.581835222Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"0","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21812024-01-10T17:56:08.489Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.58246994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"2","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21822024-01-10T17:56:08.489Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.58253664Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"2","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21832024-01-10T17:56:08.489Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.582563839Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21842024-01-10T17:56:08.489Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.582746034Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"1","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21852024-01-10T17:56:08.489Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.582819386Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"1","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21862024-01-10T17:56:08.489Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.582884484Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21872024-01-10T17:56:08.489Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.583202048Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"0","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21882024-01-10T17:56:08.489Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.583938541Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"0","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21892024-01-10T17:56:08.489Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584055613Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"0","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21902024-01-10T17:56:08.489Z{"msg":"[0]R flush_numbers[0..12]: [33, 12, 28, 32, 32, 20, 20, 21, 28, 33, 29, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584119733Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21912024-01-10T17:56:08.489Z{"msg":"[0]R generation[0..12]: [19, 8, 17, 18, 18, 13, 13, 14, 17, 19, 18, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584146897Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21922024-01-10T17:56:08.489Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584170987Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21932024-01-10T17:56:08.490Z{"msg":"[1]R flush_numbers[0..12]: [33, 12, 28, 32, 32, 20, 20, 21, 28, 33, 29, 19]","v":0,"name":Wait for a query_work_queue command to finish before sending IO
21942024-01-10T17:56:08.490Z"crucible","level":30,"time":"2024-01-10T17:55:48.584234078Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21952024-01-10T17:56:08.490Z{"msg":"[1]R generation[0..12]: [19, 8, 17, 18, 18, 13, 13, 14, 17, 19, 18, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584297488Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21962024-01-10T17:56:08.490Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584325927Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21972024-01-10T17:56:08.490Z{"msg":"[2]R flush_numbers[0..12]: [33, 12, 28, 32, 32, 20, 20, 21, 28, 33, 29, 19]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584390102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21982024-01-10T17:56:08.490Z{"msg":"[2]R generation[0..12]: [19, 8, 17, 18, 18, 13, 13, 14, 17, 19, 18, 12]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584471828Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
21992024-01-10T17:56:08.490Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584517573Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22002024-01-10T17:56:08.490Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584550757Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22012024-01-10T17:56:08.490Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584618074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22022024-01-10T17:56:08.490Z{"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.58467591Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22032024-01-10T17:56:08.491Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584719046Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"":"downstairs","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22042024-01-10T17:56:08.491Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584742503Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22052024-01-10T17:56:08.491Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584771796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"0","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22062024-01-10T17:56:08.491Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.584833446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"1","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22072024-01-10T17:56:08.492Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22082024-01-10T17:56:08.492Z,"time":"2024-01-10T17:55:48.584871793Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"client":"2","":"downstairs","session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22092024-01-10T17:56:08.492Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.585001908Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22102024-01-10T17:56:08.492Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.585072345Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22112024-01-10T17:56:08.492Z{"msg":"5d8b9143-5cc3-4fb3-a0dd-cab5e217d6d7 is now active with session: 0f3aa5af-9da1-4a31-8ecb-45599946a7f3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.585140627Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22122024-01-10T17:56:08.492Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.585204392Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235,"session_id":"0f3aa5af-9da1-4a31-8ecb-45599946a7f3"}
22132024-01-10T17:56:08.492Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:48.585268445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1235}
22142024-01-10T17:56:08.492ZLoading write count information from file "/tmp/test_repair_verify.data"
22152024-01-10T17:56:08.492ZRun Repair workload
22162024-01-10T17:56:08.492Z01/30 Flush
22172024-01-10T17:56:08.589Z02/30 Flush
22182024-01-10T17:56:08.589Z03/30 Flush
22192024-01-10T17:56:08.589Z04/30 Write block 290 len 8192 data: 3 4
22202024-01-10T17:56:08.589Z05/30 Read block 153 len 24576
22212024-01-10T17:56:08.590Z06/30 Read block 286 len 20480
22222024-01-10T17:56:08.590Z07/30 Flush
22232024-01-10T17:56:08.590Z08/30 Write block 14 len 24576 data: 5 5 5 4 4 3
22242024-01-10T17:56:08.590Z09/30 Read block 108 len 40960
22252024-01-10T17:56:08.590Z10/30 Write block 586 len 36864 data: 3 2 2 3 3 4 4 4 4
22262024-01-10T17:56:08.590Z11/30 Write block 187 len 36864 data: 3 4 4 4 3 3 3 4 4
22272024-01-10T17:56:08.590Z12/30 Read block 211 len 36864
22282024-01-10T17:56:08.590Z13/30 Flush
22292024-01-10T17:56:08.590Z14/30 Write block 236 len 12288 data: 3 4 4
22302024-01-10T17:56:08.590Z15/30 Write block 546 len 36864 data: 4 4 3 3 3 3 4 3 3
22312024-01-10T17:56:08.590Z16/30 Write block 487 len 40960 data: 3 3 2 3 3 4 4 5 6 6
22322024-01-10T17:56:08.590Z17/30 Write block 55 len 24576 data: 5 4 3 3 2 2
22332024-01-10T17:56:08.590Z18/30 Read block 274 len 4096
22342024-01-10T17:56:08.590Z19/30 Read block 581 len 16384
22352024-01-10T17:56:08.590Z20/30 Write block 486 len 40960 data: 3 4 4 3 4 4 5 5 6 7
22362024-01-10T17:56:08.590Z21/30 Read block 513 len 36864
22372024-01-10T17:56:08.590Z22/30 Flush
22382024-01-10T17:56:08.590Z23/30 Write block 10 len 8192 data: 3 3
22392024-01-10T17:56:08.591Z24/30 Read block 450 len 32768
22402024-01-10T17:56:08.591Z25/30 Read block 88 len 20480
22412024-01-10T17:56:08.591Z26/30 Read block 421 len 40960
22422024-01-10T17:56:08.591Z27/30 Write block 164 len 32768 data: 6 6 5 4 3 2 2 3
22432024-01-10T17:56:08.591Z28/30 Read block 239 len 40960
22442024-01-10T17:56:08.591Z29/30 Write block 410 len 20480 data: 2 3 3 3 3
22452024-01-10T17:56:08.591Z30/30 Read block 72 len 12288
22462024-01-10T17:56:08.591Z----------------------------------------------------------------
22472024-01-10T17:56:08.591Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:29
22482024-01-10T17:56:08.591ZStates: Active Active Active
22492024-01-10T17:56:08.591ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22502024-01-10T17:56:08.591Z 2 Acked 1001 Flush 0 Sent Done Done false
22512024-01-10T17:56:08.591Z 3 Acked 1002 Flush 0 Sent Done Done false
22522024-01-10T17:56:08.591Z 4 Acked 1003 Write 2 Sent Done Done false
22532024-01-10T17:56:08.591Z 5 Acked 1004 Read 6 Sent Done Done false
22542024-01-10T17:56:08.591Z 6 Acked 1005 Read 5 Sent Done Done false
22552024-01-10T17:56:08.591Z 7 Acked 1006 Flush 0 Sent Done Done false
22562024-01-10T17:56:08.593Z 8 Acked 1007 Write 6 Sent Done Done false
22572024-01-10T17:56:08.593Z 9 Acked 1008 Read 10 Sent Done Done false
22582024-01-10T17:56:08.593Z 10 Acked 1009 Write 9 Sent Done Done false
22592024-01-10T17:56:08.593Z 11 Acked 1010 Write 9 Sent Done Done false
22602024-01-10T17:56:08.593Z 12 Acked 1011 Read 9 Sent Done Done false
22612024-01-10T17:56:08.594Z 13 Acked 1012 Flush 0 Sent Done Done false
22622024-01-10T17:56:08.594Z 14 Acked 1013 Write 3 Sent Done Done false
22632024-01-10T17:56:08.594Z 15 Acked 1014 Write 9 Sent Done Done false
22642024-01-10T17:56:08.594Z 16 Acked 1015 Write 10 Sent Done Done false
22652024-01-10T17:56:08.595Z 17 Acked 1016 Write 6 Sent Done Done false
22662024-01-10T17:56:08.595Z 18 Acked 1017 Read 1 Sent Done Done false
22672024-01-10T17:56:08.595Z 19 Acked 1018 Read 4 Sent Done Done false
22682024-01-10T17:56:08.595Z 20 Acked 1019 Write 10 Sent Done Done false
22692024-01-10T17:56:08.595Z 21 Acked 1020 Read 9 Sent Done Done false
22702024-01-10T17:56:08.595Z 22 Acked 1021 Flush 0 Sent Done Done false
22712024-01-10T17:56:08.595Z 23 Acked 1022 Write 2 Sent Done Done false
22722024-01-10T17:56:08.595Z 24 Acked 1023 Read 8 Sent Done Done false
22732024-01-10T17:56:08.595Z 25 Acked 1024 Read 5 Sent Done Done false
22742024-01-10T17:56:08.595Z 26 Acked 1025 Read 10 Sent Done Done false
22752024-01-10T17:56:08.595Z 27 Acked 1026 Write 8 Sent Done Done false
22762024-01-10T17:56:08.595Z 28 Acked 1027 Read 10 Sent Done Done false
22772024-01-10T17:56:08.595Z 29 Acked 1028 Write 5 Sent Done Done false
22782024-01-10T17:56:08.595Z 30 Acked 1029 Read 3 Sent Done Done false
22792024-01-10T17:56:08.595Z STATES DS:0 DS:1 DS:2 TOTAL
22802024-01-10T17:56:08.595Z New 0 0 0 0
22812024-01-10T17:56:08.595Z Sent 29 0 0 29
22822024-01-10T17:56:08.595Z Done 0 29 29 58
22832024-01-10T17:56:08.596Z Skipped 0 0 0 0
22842024-01-10T17:56:08.596Z Error 0 0 0 0
22852024-01-10T17:56:08.596ZLast Flush: 1000 1021 1021
22862024-01-10T17:56:08.597ZDownstairs last five completed: 1000
22872024-01-10T17:56:08.597ZUpstairs last five completed: 30 29 28 27 26
22882024-01-10T17:56:08.597ZWrote out file "/tmp/test_repair_verify.data"
22892024-01-10T17:56:08.597Z
22902024-01-10T17:56:08.597ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
22912024-01-10T17:56:08.597ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
22922024-01-10T17:56:08.597Z 0 000-019 19 20 20 33 38 38 F T T <---
22932024-01-10T17:56:08.597Z 1 020-039 8 8 8 12 12 12 F F F
22942024-01-10T17:56:08.597Z 2 040-059 17 20 20 28 39 39 F F F <---
22952024-01-10T17:56:08.597Z 3 060-079 18 20 20 32 39 39 F F F <---
22962024-01-10T17:56:08.597Z 4 080-099 18 18 18 32 32 32 F F F
22972024-01-10T17:56:08.597Z 5 100-119 13 13 13 20 20 20 F F F
22982024-01-10T17:56:08.597Z 6 120-139 13 13 13 20 20 20 F F F
22992024-01-10T17:56:08.597Z 7 140-159 14 14 14 21 21 21 F F F
23002024-01-10T17:56:08.597Z 8 160-179 17 17 17 28 28 28 F T T <---
23012024-01-10T17:56:08.597Z 9 180-199 19 20 20 33 38 38 F F F <---
23022024-01-10T17:56:08.597Z 10 200-219 18 18 18 29 29 29 F F F
23032024-01-10T17:56:08.597Z 11 220-239 12 20 20 19 39 39 F F F <---
23042024-01-10T17:56:08.598Z 12 240-259 16 16 16 27 27 27 F F F
23052024-01-10T17:56:08.598Z 13 260-279 19 19 19 33 33 33 F F F
23062024-01-10T17:56:08.598Z 14 280-299 19 20 20 33 37 37 F F F <---
23072024-01-10T17:56:08.598Z 15 300-319 16 16 16 25 25 25 F F F
23082024-01-10T17:56:08.598Z 16 320-339 16 16 16 25 25 25 F F F
23092024-01-10T17:56:08.598Z 17 340-359 12 12 12 19 19 19 F F F
23102024-01-10T17:56:08.598Z 18 360-379 13 13 13 20 20 20 F F F
23112024-01-10T17:56:08.599Z 19 380-399 19 19 19 33 33 33 F F F
23122024-01-10T17:56:08.599Z 20 400-419 15 15 15 22 22 22 F T T <---
23132024-01-10T17:56:08.599Z 21 420-439 19 19 19 33 33 33 F F F
23142024-01-10T17:56:08.599Z 22 440-459 18 18 18 32 32 32 F F F
23152024-01-10T17:56:08.599Z 23 460-479 19 19 19 33 33 33 F F F
23162024-01-10T17:56:08.599Z 24 480-499 18 20 20 32 39 39 F F F <---
23172024-01-10T17:56:08.875Z 25 500-519 18 18 18 30 30 30 F F F
23182024-01-10T17:56:08.875Z 26 520-539 11 11 11 18 18 18 F F F
23192024-01-10T17:56:08.875Z 27 540-559 16 20 20 26 39 39 F F F <---
23202024-01-10T17:56:08.875Z 28 560-579 18 18 18 32 32 32 F F F
23212024-01-10T17:56:08.875Z 29 580-599 13 20 20 20 38 38 F F F <---
23222024-01-10T17:56:08.875ZMax gen: 20, Max flush: 39
23232024-01-10T17:56:08.875ZError: Difference in extent metadata found!
23242024-01-10T17:56:08.875ZA Difference in extent metadata is expected here
23252024-01-10T17:56:08.876ZOn loop 10
23262024-01-10T17:56:08.876Z
23272024-01-10T17:56:08.876ZVerifying data now
23282024-01-10T17:56:08.967ZNo change after verify
23292024-01-10T17:56:08.967ZLoop: 10 Downstairs dump after verify (and repair):
23302024-01-10T17:56:08.967ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23312024-01-10T17:56:08.967Z 0 000-019 21 21 21 40 40 40 F F F
23322024-01-10T17:56:08.967Z 1 020-039 8 8 8 12 12 12 F F F
23332024-01-10T17:56:08.967Z 2 040-059 20 20 20 39 39 39 F F F
23342024-01-10T17:56:08.968Z 3 060-079 20 20 20 39 39 39 F F F
23352024-01-10T17:56:08.968Z 4 080-099 18 18 18 32 32 32 F F F
23362024-01-10T17:56:08.968Z 5 100-119 13 13 13 20 20 20 F F F
23372024-01-10T17:56:08.968Z 6 120-139 13 13 13 20 20 20 F F F
23382024-01-10T17:56:08.968Z 7 140-159 14 14 14 21 21 21 F F F
23392024-01-10T17:56:08.968Z 8 160-179 21 21 21 40 40 40 F F F
23402024-01-10T17:56:08.968Z 9 180-199 20 20 20 38 38 38 F F F
23412024-01-10T17:56:08.968Z 10 200-219 18 18 18 29 29 29 F F F
23422024-01-10T17:56:08.968Z 11 220-239 20 20 20 39 39 39 F F F
23432024-01-10T17:56:08.968Z 12 240-259 16 16 16 27 27 27 F F F
23442024-01-10T17:56:08.969Z 13 260-279 19 19 19 33 33 33 F F F
23452024-01-10T17:56:08.969Z 14 280-299 20 20 20 37 37 37 F F F
23462024-01-10T17:56:08.969Z 15 300-319 16 16 16 25 25 25 F F F
23472024-01-10T17:56:08.969Z 16 320-339 16 16 16 25 25 25 F F F
23482024-01-10T17:56:08.969Z 17 340-359 12 12 12 19 19 19 F F F
23492024-01-10T17:56:08.970Z 18 360-379 13 13 13 20 20 20 F F F
23502024-01-10T17:56:08.970Z 19 380-399 19 19 19 33 33 33 F F F
23512024-01-10T17:56:08.970Z 20 400-419 21 21 21 40 40 40 F F F
23522024-01-10T17:56:08.970Z 21 420-439 19 19 19 33 33 33 F F F
23532024-01-10T17:56:08.970Z 22 440-459 18 18 18 32 32 32 F F F
23542024-01-10T17:56:08.970Z 23 460-479 19 19 19 33 33 33 F F F
23552024-01-10T17:56:08.970Z 24 480-499 20 20 20 39 39 39 F F F
23562024-01-10T17:56:08.970Z 25 500-519 18 18 18 30 30 30 F F F
23572024-01-10T17:56:08.970Z 26 520-539 11 11 11 18 18 18 F F F
23582024-01-10T17:56:08.970Z 27 540-559 20 20 20 39 39 39 F F F
23592024-01-10T17:56:08.970Z 28 560-579 18 18 18 32 32 32 F F F
23602024-01-10T17:56:08.970Z 29 580-599 20 20 20 38 38 38 F F F
23612024-01-10T17:56:08.970ZMax gen: 21, Max flush: 40
23622024-01-10T17:56:08.970Z
23632024-01-10T17:56:08.970ZBegin loop 11
23642024-01-10T17:56:08.970ZDownstairs to restart: 2
23652024-01-10T17:56:08.970Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.866965637Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243}
23662024-01-10T17:56:08.970Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.867474558Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243}
23672024-01-10T17:56:08.971Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.867558259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243}
23682024-01-10T17:56:08.971Z{Crucible runtime is spawned
23692024-01-10T17:56:08.972Z"msg":"Crucible 1f1cbb41-f68b-4a01-a315-37f1901a872e has session id: 2426ae13-d697-41c4-a918-569aa52436e3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.867606002Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23702024-01-10T17:56:08.972Z{"msg":"Upstairs opts: Upstairs UUID: 1f1cbb41-f68b-4a01-a315-37f1901a872e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.867740568Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23712024-01-10T17:56:08.972Z{"msg":"Crucible stats registered with UUID: 1f1cbb41-f68b-4a01-a315-37f1901a872e","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.867786409Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23722024-01-10T17:56:08.972Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.867832819Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243}
23732024-01-10T17:56:08.972Z{"msg":"1f1cbb41-f68b-4a01-a315-37f1901a872e active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868178668Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23742024-01-10T17:56:08.972Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868255125Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"0","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23752024-01-10T17:56:08.972Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868335109Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"0","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23762024-01-10T17:56:08.972Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868408484Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"1","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23772024-01-10T17:56:08.972Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868463961Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"1","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23782024-01-10T17:56:08.972Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868506949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"2","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23792024-01-10T17:56:08.972Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868604062Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"2","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23802024-01-10T17:56:08.972Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868715003Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23812024-01-10T17:56:08.972Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868799793Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23822024-01-10T17:56:08.972Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868877674Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23832024-01-10T17:56:08.972Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.868956911Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23842024-01-10T17:56:08.972Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.869150747Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23852024-01-10T17:56:08.972Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.869341566Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23862024-01-10T17:56:08.972Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.870081336Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"0","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23872024-01-10T17:56:08.972Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.870182905Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"0","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23882024-01-10T17:56:08.972Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.870247479Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"1","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23892024-01-10T17:56:08.972Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.870446281Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"1","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23902024-01-10T17:56:08.972Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.87115115Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"1","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23912024-01-10T17:56:08.973Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.871463261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"2","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23922024-01-10T17:56:08.973Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.871521264Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"2","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23932024-01-10T17:56:08.973Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.871597727Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"0","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23942024-01-10T17:56:08.973Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.872014123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"1","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23952024-01-10T17:56:08.973Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.87214952Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"1","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23962024-01-10T17:56:08.973Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.872224356Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23972024-01-10T17:56:08.974Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.87248094Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"0","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23982024-01-10T17:56:08.974Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.872539645Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"0","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
23992024-01-10T17:56:08.974Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.872582653Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24002024-01-10T17:56:08.974Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.872622581Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"2","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24012024-01-10T17:56:08.974Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873257664Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"2","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24022024-01-10T17:56:08.974Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873332218Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"2","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24032024-01-10T17:56:08.974Z{"msg":"[0]R flush_numbers[0..12]: [40, 12, 39, 39, 32, 20, 20, 21, 40, 38, 29, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873397894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24042024-01-10T17:56:08.974Z{"msg":"[0]R generation[0..12]: [21, 8, 20, 20, 18, 13, 13, 14, 21, 20, 18, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.87344461Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24052024-01-10T17:56:08.974Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.87349602Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24062024-01-10T17:56:08.974ZWait for a query_work_queue command to finish before sending IO
24072024-01-10T17:56:08.975Z{"msg":"[1]R flush_numbers[0..12]: [40, 12, 39, 39, 32, 20, 20, 21, 40, 38, 29, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873545642Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24082024-01-10T17:56:08.975Z{"msg":"[1]R generation[0..12]: [21, 8, 20, 20, 18, 13, 13, 14, 21, 20, 18, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873596991Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24092024-01-10T17:56:08.975Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873639436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24102024-01-10T17:56:08.975Z{"msg":"[2]R flush_numbers[0..12]: [40, 12, 39, 39, 32, 20, 20, 21, 40, 38, 29, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873684871Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24112024-01-10T17:56:08.975Z{"msg":"[2]R generation[0..12]: [21, 8, 20, 20, 18, 13, 13, 14, 21, 20, 18, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873725579Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24122024-01-10T17:56:08.975Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873771104Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24132024-01-10T17:56:08.975Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.8738192Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24142024-01-10T17:56:08.975Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873842999Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24152024-01-10T17:56:08.975Z{"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873884834Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24162024-01-10T17:56:08.975Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873928679Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"":"downstairs","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24172024-01-10T17:56:08.976Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873951591Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24182024-01-10T17:56:08.976Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.873991878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"0","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24192024-01-10T17:56:08.976Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.874014865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"1","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24202024-01-10T17:56:08.976Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.874054529Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"client":"2","":"downstairs","session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24212024-01-10T17:56:08.976Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.874104743Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24222024-01-10T17:56:08.976Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.874144002Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24232024-01-10T17:56:08.976Z{"msg":"1f1cbb41-f68b-4a01-a315-37f1901a872e is now active with session: 2426ae13-d697-41c4-a918-569aa52436e3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.874184149Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24242024-01-10T17:56:08.976Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24252024-01-10T17:56:08.977Z2024-01-10T17:55:53.874231042Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243,"session_id":"2426ae13-d697-41c4-a918-569aa52436e3"}
24262024-01-10T17:56:08.977Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:55:53.874260095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1243}
24272024-01-10T17:56:08.977ZLoading write count information from file "/tmp/test_repair_verify.data"
24282024-01-10T17:56:09.069ZRun Repair workload
24292024-01-10T17:56:09.069ZError: Difference in extent metadata found!
24302024-01-10T17:56:09.069Z01/30 Write block 293 len 40960 data: 3 2 2 3 3 4 4 4 4 3
24312024-01-10T17:56:09.069Z02/30 Flush
24322024-01-10T17:56:09.069Z03/30 Write block 579 len 20480 data: 3 3 3 4 4
24332024-01-10T17:56:09.069Z04/30 Read block 164 len 36864
24342024-01-10T17:56:09.069Z05/30 Read block 2 len 32768
24352024-01-10T17:56:09.069Z06/30 Read block 270 len 12288
24362024-01-10T17:56:09.069Z07/30 Read block 494 len 32768
24372024-01-10T17:56:09.070Z08/30 Read block 140 len 20480
24382024-01-10T17:56:09.070Z09/30 Write block 14 len 36864 data: 6 6 6 5 5 4 3 4 3
24392024-01-10T17:56:09.070Z10/30 Write block 371 len 16384 data: 2 2 2 2
24402024-01-10T17:56:09.070Z11/30 Read block 85 len 36864
24412024-01-10T17:56:09.070Z12/30 Read block 3 len 24576
24422024-01-10T17:56:09.070Z13/30 Read block 141 len 32768
24432024-01-10T17:56:09.071Z14/30 Read block 79 len 4096
24442024-01-10T17:56:09.071Z15/30 Read block 117 len 12288
24452024-01-10T17:56:09.071Z16/30 Read block 111 len 16384
24462024-01-10T17:56:09.071Z17/30 Read block 119 len 8192
24472024-01-10T17:56:09.071Z18/30 Read block 398 len 8192
24482024-01-10T17:56:09.071Z19/30 Flush
24492024-01-10T17:56:09.071Z20/30 Flush
24502024-01-10T17:56:09.071Z21/30 Write block 120 len 24576 data: 4 4 3 2 2 2
24512024-01-10T17:56:09.072Z22/30 Flush
24522024-01-10T17:56:09.072Z23/30 Write block 112 len 8192 data: 3 3
24532024-01-10T17:56:09.072Z24/30 Read block 167 len 16384
24542024-01-10T17:56:09.072Z25/30 Read block 291 len 24576
24552024-01-10T17:56:09.072Z26/30 Write block 407 len 32768 data: 2 2 2 3 4 4 4 4
24562024-01-10T17:56:09.072Z27/30 Write block 328 len 16384 data: 4 3 3 2
24572024-01-10T17:56:09.072Z28/30 Write block 177 len 28672 data: 3 3 2 2 2 2 2
24582024-01-10T17:56:09.072Z29/30 Read block 541 len 4096
24592024-01-10T17:56:09.072Z30/30 Read block 412 len 8192
24602024-01-10T17:56:09.072Z----------------------------------------------------------------
24612024-01-10T17:56:09.072Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:28
24622024-01-10T17:56:09.072ZStates: Active Active Active
24632024-01-10T17:56:09.072ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24642024-01-10T17:56:09.072Z 3 Acked 1002 Write 5 Done Done Done false
24652024-01-10T17:56:09.072Z 4 Acked 1003 Read 9 Done Done Done false
24662024-01-10T17:56:09.072Z 5 Acked 1004 Read 8 Done Done Done false
24672024-01-10T17:56:09.072Z 6 Acked 1005 Read 3 Done Done Done false
24682024-01-10T17:56:09.072Z 7 Acked 1006 Read 8 Done Done Done false
24692024-01-10T17:56:09.072Z 8 Acked 1007 Read 5 Done Done Done false
24702024-01-10T17:56:09.074Z 9 Acked 1008 Write 9 Done Done Done false
24712024-01-10T17:56:09.074Z 10 Acked 1009 Write 4 Done Done Done false
24722024-01-10T17:56:09.074Z 11 Acked 1010 Read 9 Done Done Done false
24732024-01-10T17:56:09.074Z 12 Acked 1011 Read 6 Done Done Sent false
24742024-01-10T17:56:09.074Z 13 Acked 1012 Read 8 Done Done Sent false
24752024-01-10T17:56:09.074Z 14 Acked 1013 Read 1 Done Done Sent false
24762024-01-10T17:56:09.074Z 15 Acked 1014 Read 3 Done Done Sent false
24772024-01-10T17:56:09.074Z 16 Acked 1015 Read 4 Done Done Sent false
24782024-01-10T17:56:09.074Z 17 Acked 1016 Read 2 Done Done Sent false
24792024-01-10T17:56:09.074Z 18 Acked 1017 Read 2 Done Done Sent false
24802024-01-10T17:56:09.074Z 19 Acked 1018 Flush 0 Done Done Sent false
24812024-01-10T17:56:09.074Z 20 Acked 1019 Flush 0 Done Done Sent false
24822024-01-10T17:56:09.074Z 21 Acked 1020 Write 6 Done Done Sent false
24832024-01-10T17:56:09.074Z 22 Acked 1021 Flush 0 Done Done Sent false
24842024-01-10T17:56:09.074Z 23 Acked 1022 Write 2 Done Done Sent false
24852024-01-10T17:56:09.075Z 24 Acked 1023 Read 4 Done Done Sent false
24862024-01-10T17:56:09.075Z 25 Acked 1024 Read 6 Done Done Sent false
24872024-01-10T17:56:09.075Z 26 Acked 1025 Write 8 Done Done Sent false
24882024-01-10T17:56:09.075Z 27 Acked 1026 Write 4 Done Done Sent false
24892024-01-10T17:56:09.075Z 28 Acked 1027 Write 7 Done Done Sent false
24902024-01-10T17:56:09.075Z 29 Acked 1028 Read 1 Done Done Sent false
24912024-01-10T17:56:09.075Z 30 Acked 1029 Read 2 Done Done Sent false
24922024-01-10T17:56:09.075Z STATES DS:0 DS:1 DS:2 TOTAL
24932024-01-10T17:56:09.075Z New 0 0 0 0
24942024-01-10T17:56:09.075Z Sent 0 0 19 19
24952024-01-10T17:56:09.076Z Done 28 28 9 65
24962024-01-10T17:56:09.076Z Skipped 0 0 0 0
24972024-01-10T17:56:09.076Z Error 0 0 0 0
24982024-01-10T17:56:09.076ZLast Flush: 1021 1021 1001
24992024-01-10T17:56:09.076ZDownstairs last five completed: 1001 1000
25002024-01-10T17:56:09.076ZUpstairs last five completed: 30 29 28 27 26
25012024-01-10T17:56:09.076ZWrote out file "/tmp/test_repair_verify.data"
25022024-01-10T17:56:09.076Z
25032024-01-10T17:56:09.076ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25042024-01-10T17:56:09.076ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25052024-01-10T17:56:09.076Z 0 000-019 22 22 21 42 42 40 F F T <---
25062024-01-10T17:56:09.076Z 1 020-039 22 22 8 42 42 12 F F T <---
25072024-01-10T17:56:09.076Z 2 040-059 20 20 20 39 39 39 F F F
25082024-01-10T17:56:09.076Z 3 060-079 20 20 20 39 39 39 F F F
25092024-01-10T17:56:09.076Z 4 080-099 18 18 18 32 32 32 F F F
25102024-01-10T17:56:09.076Z 5 100-119 13 13 13 20 20 20 T T F <---
25112024-01-10T17:56:09.076Z 6 120-139 22 22 13 44 44 20 F F F <---
25122024-01-10T17:56:09.076Z 7 140-159 14 14 14 21 21 21 F F F
25132024-01-10T17:56:09.076Z 8 160-179 21 21 21 40 40 40 T T F <---
25142024-01-10T17:56:09.076Z 9 180-199 20 20 20 38 38 38 T T F <---
25152024-01-10T17:56:09.076Z 10 200-219 18 18 18 29 29 29 F F F
25162024-01-10T17:56:09.076Z 11 220-239 20 20 20 39 39 39 F F F
25172024-01-10T17:56:09.076Z 12 240-259 16 16 16 27 27 27 F F F
25182024-01-10T17:56:09.077Z 13 260-279 19 19 19 33 33 33 F F F
25192024-01-10T17:56:09.077Z 14 280-299 22 22 22 41 41 41 F F F
25202024-01-10T17:56:09.077Z 15 300-319 22 22 22 41 41 41 F F F
25212024-01-10T17:56:09.077Z 16 320-339 16 16 16 25 25 25 T T F <---
25222024-01-10T17:56:09.077Z 17 340-359 12 12 12 19 19 19 F F F
25232024-01-10T17:56:09.077Z 18 360-379 22 22 13 42 42 20 F F T <---
25242024-01-10T17:56:09.077Z 19 380-399 19 19 19 33 33 33 F F F
25252024-01-10T17:56:09.077Z 20 400-419 21 21 21 40 40 40 T T F <---
25262024-01-10T17:56:09.077Z 21 420-439 19 19 19 33 33 33 F F F
25272024-01-10T17:56:09.077Z 22 440-459 18 18 18 32 32 32 F F F
25282024-01-10T17:56:09.118Z 23 460-479 19 19 19 33 33 33 F F F
25292024-01-10T17:56:09.118Z 24 480-499 20 20 20 39 39 39 F F F
25302024-01-10T17:56:09.118Z 25 500-519 18 18 18 30 30 30 F F F
25312024-01-10T17:56:09.119Z 26 520-539 11 11 11 18 18 18 F F F
25322024-01-10T17:56:09.119Z 27 540-559 20 20 20 39 39 39 F F F
25332024-01-10T17:56:09.119Z 28 560-579 22 22 18 42 42 32 F F T <---
25342024-01-10T17:56:09.119Z 29 580-599 22 22 20 42 42 38 F F T <---
25352024-01-10T17:56:09.120ZMax gen: 22, Max flush: 44
25362024-01-10T17:56:09.120ZA Difference in extent metadata is expected here
25372024-01-10T17:56:09.120ZOn loop 11
25382024-01-10T17:56:09.121Z
25392024-01-10T17:56:09.121ZVerifying data now
25402024-01-10T17:56:09.121ZNo change after verify
25412024-01-10T17:56:09.121ZLoop: 11 Downstairs dump after verify (and repair):
25422024-01-10T17:56:09.121ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25432024-01-10T17:56:09.122Z 0 000-019 22 22 22 42 42 42 F F F
25442024-01-10T17:56:09.123Z 1 020-039 22 22 22 42 42 42 F F F
25452024-01-10T17:56:09.123Z 2 040-059 20 20 20 39 39 39 F F F
25462024-01-10T17:56:09.123Z 3 060-079 20 20 20 39 39 39 F F F
25472024-01-10T17:56:09.123Z 4 080-099 18 18 18 32 32 32 F F F
25482024-01-10T17:56:09.124Z 5 100-119 23 23 23 45 45 45 F F F
25492024-01-10T17:56:09.124Z 6 120-139 22 22 22 44 44 44 F F F
25502024-01-10T17:56:09.124Z 7 140-159 14 14 14 21 21 21 F F F
25512024-01-10T17:56:09.124Z 8 160-179 23 23 23 45 45 45 F F F
25522024-01-10T17:56:09.124Z 9 180-199 23 23 23 45 45 45 F F F
25532024-01-10T17:56:09.124Z 10 200-219 18 18 18 29 29 29 F F F
25542024-01-10T17:56:09.124Z 11 220-239 20 20 20 39 39 39 F F F
25552024-01-10T17:56:09.125Z 12 240-259 16 16 16 27 27 27 F F F
25562024-01-10T17:56:09.125Z 13 260-279 19 19 19 33 33 33 F F F
25572024-01-10T17:56:09.125Z 14 280-299 22 22 22 41 41 41 F F F
25582024-01-10T17:56:09.125Z 15 300-319 22 22 22 41 41 41 F F F
25592024-01-10T17:56:09.126Z 16 320-339 23 23 23 45 45 45 F F F
25602024-01-10T17:56:09.126Z 17 340-359 12 12 12 19 19 19 F F F
25612024-01-10T17:56:09.126Z 18 360-379 22 22 22 42 42 42 F F F
25622024-01-10T17:56:09.126Z 19 380-399 19 19 19 33 33 33 F F F
25632024-01-10T17:56:09.126Z 20 400-419 23 23 23 45 45 45 F F F
25642024-01-10T17:56:09.126Z 21 420-439 19 19 19 33 33 33 F F F
25652024-01-10T17:56:09.126Z 22 440-459 18 18 18 32 32 32 F F F
25662024-01-10T17:56:09.126Z 23 460-479 19 19 19 33 33 33 F F F
25672024-01-10T17:56:09.127Z 24 480-499 20 20 20 39 39 39 F F F
25682024-01-10T17:56:09.127Z 25 500-519 18 18 18 30 30 30 F F F
25692024-01-10T17:56:09.127Z 26 520-539 11 11 11 18 18 18 F F F
25702024-01-10T17:56:09.127Z 27 540-559 20 20 20 39 39 39 F F F
25712024-01-10T17:56:09.128Z 28 560-579 22 22 22 42 42 42 F F F
25722024-01-10T17:56:09.128Z 29 580-599 22 22 22 42 42 42 F F F
25732024-01-10T17:56:09.128ZMax gen: 23, Max flush: 45
25742024-01-10T17:56:09.128Z
25752024-01-10T17:56:09.128ZBegin loop 12
25762024-01-10T17:56:09.128ZDownstairs to restart: 0
25772024-01-10T17:56:09.128Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.168448929Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251}
25782024-01-10T17:56:09.128Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.168926768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251}
25792024-01-10T17:56:09.128Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.168954919Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251}
25802024-01-10T17:56:09.128Z{"msg":"Crucible e8afd912-4382-40d5-ba6f-0d248dcf6ca1 has session id: 607e30a7-b299-4ea8-8a5e-71177f9638a4","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.168975416Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25812024-01-10T17:56:09.128Z{"msg":"Upstairs opts: Upstairs UUID: e8afd912-4382-40d5-ba6f-0d248dcf6ca1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169009261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25822024-01-10T17:56:09.129Z{"msg":"Crucible stats registered with UUID: e8afd912-4382-40d5-ba6f-0d248dcf6ca1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169032409Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25832024-01-10T17:56:09.129ZCrucible runtime is spawned
25842024-01-10T17:56:09.130Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169363001Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251}
25852024-01-10T17:56:09.130Z{"msg":"e8afd912-4382-40d5-ba6f-0d248dcf6ca1 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169574625Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25862024-01-10T17:56:09.130Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.16961509Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"0","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25872024-01-10T17:56:09.130Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169640492Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"0","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25882024-01-10T17:56:09.130Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169663416Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"1","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25892024-01-10T17:56:09.130Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169685754Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"1","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25902024-01-10T17:56:09.130Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169707836Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"2","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25912024-01-10T17:56:09.131Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169730242Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"2","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25922024-01-10T17:56:09.131Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169751708Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25932024-01-10T17:56:09.131Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.16977435Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25942024-01-10T17:56:09.131Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169796894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25952024-01-10T17:56:09.131Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.169991231Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25962024-01-10T17:56:09.131Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.170176337Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25972024-01-10T17:56:09.132Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.170231585Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25982024-01-10T17:56:09.132Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.171407386Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"1","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
25992024-01-10T17:56:09.132Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.17147039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"1","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26002024-01-10T17:56:09.132Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.171520034Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"2","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26012024-01-10T17:56:09.132Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.171567408Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"2","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26022024-01-10T17:56:09.132Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.171752073Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"0","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26032024-01-10T17:56:09.133Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.171787373Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"0","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26042024-01-10T17:56:09.133Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.172604634Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"1","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26052024-01-10T17:56:09.133Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.172647055Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"2","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26062024-01-10T17:56:09.133Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.172767023Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"0","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26072024-01-10T17:56:09.133Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.173496995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"1","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26082024-01-10T17:56:09.133Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.173544559Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"1","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26092024-01-10T17:56:09.134Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.173569737Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26102024-01-10T17:56:09.134Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.173722855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"2","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26112024-01-10T17:56:09.134Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:00.173757173Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"2","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26122024-01-10T17:56:09.134Z{Wait for a query_work_queue command to finish before sending IO
26132024-01-10T17:56:09.135Z"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.609868842Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26142024-01-10T17:56:09.136Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.6099281Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"0","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26152024-01-10T17:56:09.136Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.609953167Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"0","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26162024-01-10T17:56:09.137Z{"msg":"[0]R flush_numbers[0..12]: [42, 42, 39, 39, 32, 45, 44, 21, 45, 45, 29, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.609976454Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26172024-01-10T17:56:09.137Z{"msg":"[0]R generation[0..12]: [22, 22, 20, 20, 18, 23, 22, 14, 23, 23, 18, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.609999964Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26182024-01-10T17:56:09.137Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610023691Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26192024-01-10T17:56:09.137Z{"msg":"[1]R flush_numbers[0..12]: [42, 42, 39, 39, 32, 45, 44, 21, 45, 45, 29, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610047151Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26202024-01-10T17:56:09.137Z{"msg":"[1]R generation[0..12]: [22, 22, 20, 20, 18, 23, 22, 14, 23, 23, 18, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.6100697Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26212024-01-10T17:56:09.137Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610093419Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26222024-01-10T17:56:09.137Z{"msg":"[2]R flush_numbers[0..12]: [42, 42, 39, 39, 32, 45, 44, 21, 45, 45, 29, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.61011655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26232024-01-10T17:56:09.139Z{"msg":"[2]R generation[0..12]: [22, 22, 20, 20, 18, 23, 22, 14, 23, 23, 18, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610161445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26242024-01-10T17:56:09.139Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.61020539Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26252024-01-10T17:56:09.139Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610251822Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26262024-01-10T17:56:09.139Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610298929Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26272024-01-10T17:56:09.139Z{"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610323168Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26282024-01-10T17:56:09.174Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610369656Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"":"downstairs","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26292024-01-10T17:56:09.174Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610418704Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26302024-01-10T17:56:09.174Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610460462Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"0","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26312024-01-10T17:56:09.174Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610501694Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"1","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26322024-01-10T17:56:09.174Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610543243Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"client":"2","":"downstairs","session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26332024-01-10T17:56:09.176Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.61059491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26342024-01-10T17:56:09.176Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610635881Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26352024-01-10T17:56:09.176ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26362024-01-10T17:56:09.177Z{"msg":"e8afd912-4382-40d5-ba6f-0d248dcf6ca1 is now active with session: 607e30a7-b299-4ea8-8a5e-71177f9638a4","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610681258Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26372024-01-10T17:56:09.177Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.610720353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251,"session_id":"607e30a7-b299-4ea8-8a5e-71177f9638a4"}
26382024-01-10T17:56:09.177Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:08.61075999Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1251}
26392024-01-10T17:56:09.177ZLoading write count information from file "/tmp/test_repair_verify.data"
26402024-01-10T17:56:09.177ZRun Repair workload
26412024-01-10T17:56:09.177Z01/30 Write block 91 len 40960 data: 3 2 2 2 2 2 3 3 3 3
26422024-01-10T17:56:09.177Z02/30 Flush
26432024-01-10T17:56:09.177Z03/30 Write block 110 len 12288 data: 3 3 4
26442024-01-10T17:56:09.177Z04/30 Read block 240 len 28672
26452024-01-10T17:56:09.177Z05/30 Read block 582 len 28672
26462024-01-10T17:56:09.177Z06/30 Read block 544 len 40960
26472024-01-10T17:56:09.178Z07/30 Write block 561 len 36864 data: 2 2 2 3 4 3 3 4 4
26482024-01-10T17:56:09.178Z08/30 Flush
26492024-01-10T17:56:09.178Z09/30 Write block 15 len 20480 data: 7 7 6 6 5
26502024-01-10T17:56:09.178Z10/30 Write block 23 len 40960 data: 3 3 3 3 3 3 4 4 3 3
26512024-01-10T17:56:09.178Z11/30 Read block 362 len 12288
26522024-01-10T17:56:09.178Z12/30 Read block 68 len 4096
26532024-01-10T17:56:09.178Z13/30 Write block 79 len 20480 data: 3 3 4 3 3
26542024-01-10T17:56:09.178Z14/30 Flush
26552024-01-10T17:56:09.178Z15/30 Write block 105 len 8192 data: 2 2
26562024-01-10T17:56:09.178Z16/30 Read block 283 len 20480
26572024-01-10T17:56:09.179Z17/30 Write block 512 len 12288 data: 6 7 6
26582024-01-10T17:56:09.179Z18/30 Read block 11 len 32768
26592024-01-10T17:56:09.179Z19/30 Write block 211 len 16384 data: 3 3 3 3
26602024-01-10T17:56:09.179Z20/30 Read block 425 len 16384
26612024-01-10T17:56:09.179Z21/30 Read block 544 len 32768
26622024-01-10T17:56:09.179Z22/30 Flush
26632024-01-10T17:56:09.179Z23/30 Write block 332 len 24576 data: 2 2 3 5 5 6
26642024-01-10T17:56:09.179Z24/30 Read block 573 len 12288
26652024-01-10T17:56:09.179Z25/30 Read block 272 len 40960
26662024-01-10T17:56:09.179Z26/30 Write block 395 len 28672 data: 3 3 3 3 3 3 3
26672024-01-10T17:56:09.179Z27/30 Write block 2 len 12288 data: 3 3 3
26682024-01-10T17:56:09.179Z28/30 Write block 468 len 40960 data: 6 4 4 2 2 3 3 2 2 2
26692024-01-10T17:56:09.179Z29/30 Read block 557 len 40960
26702024-01-10T17:56:09.179Z30/30 Read block 38 len 28672
26712024-01-10T17:56:09.185Z----------------------------------------------------------------
26722024-01-10T17:56:09.185Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:28
26732024-01-10T17:56:09.185ZStates: Active Active Active
26742024-01-10T17:56:09.185ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26752024-01-10T17:56:09.185Z 3 Acked 1002 Write 3 Done Done Done false
26762024-01-10T17:56:09.186Z 4 Acked 1003 Read 7 Sent Done Done false
26772024-01-10T17:56:09.186Z 5 Acked 1004 Read 7 Sent Done Done false
26782024-01-10T17:56:09.186Z 6 Acked 1005 Read 10 Sent Done Done false
26792024-01-10T17:56:09.186Z 7 Acked 1006 Write 9 Sent Done Done false
26802024-01-10T17:56:09.186Z 8 Acked 1007 Flush 0 Sent Done Done false
26812024-01-10T17:56:09.186Z 9 Acked 1008 Write 5 Sent Done Done false
26822024-01-10T17:56:09.186Z 10 Acked 1009 Write 10 Sent Done Done false
26832024-01-10T17:56:09.186Z 11 Acked 1010 Read 3 Sent Done Done false
26842024-01-10T17:56:09.186Z 12 Acked 1011 Read 1 Sent Done Done false
26852024-01-10T17:56:09.186Z 13 Acked 1012 Write 5 Sent Done Done false
26862024-01-10T17:56:09.186Z 14 Acked 1013 Flush 0 Sent Done Done false
26872024-01-10T17:56:09.186Z 15 Acked 1014 Write 2 Sent Done Done false
26882024-01-10T17:56:09.187Z 16 Acked 1015 Read 5 Sent Done Done false
26892024-01-10T17:56:09.187Z 17 Acked 1016 Write 3 Sent Done Done false
26902024-01-10T17:56:09.187Z 18 Acked 1017 Read 8 Sent Done Done false
26912024-01-10T17:56:09.187Z 19 Acked 1018 Write 4 Sent Done Done false
26922024-01-10T17:56:09.187Z 20 Acked 1019 Read 4 Sent Done Done false
26932024-01-10T17:56:09.187Z 21 Acked 1020 Read 8 Sent Done Done false
26942024-01-10T17:56:09.187Z 22 Acked 1021 Flush 0 Sent Done Done false
26952024-01-10T17:56:09.187Z 23 Acked 1022 Write 6 Sent Done Done false
26962024-01-10T17:56:09.187Z 24 Acked 1023 Read 3 Sent Done Done false
26972024-01-10T17:56:09.187Z 25 Acked 1024 Read 10 Sent Done Done false
26982024-01-10T17:56:09.187Z 26 Acked 1025 Write 7 Sent Done Done false
26992024-01-10T17:56:09.188Z 27 Acked 1026 Write 3 Sent Done Done false
27002024-01-10T17:56:09.188Z 28 Acked 1027 Write 10 Sent Done Done false
27012024-01-10T17:56:09.188Z 29 Acked 1028 Read 10 Sent Done Done false
27022024-01-10T17:56:09.188Z 30 Acked 1029 Read 7 Sent Done Done false
27032024-01-10T17:56:09.188Z STATES DS:0 DS:1 DS:2 TOTAL
27042024-01-10T17:56:09.188Z New 0 0 0 0
27052024-01-10T17:56:09.188Z Sent 27 0 0 27
27062024-01-10T17:56:09.189Z Done 1 28 28 57
27072024-01-10T17:56:09.189Z Skipped 0 0 0 0
27082024-01-10T17:56:09.189Z Error 0 0 0 0
27092024-01-10T17:56:09.189ZLast Flush: 1001 1021 1021
27102024-01-10T17:56:09.190ZDownstairs last five completed: 1001 1000
27112024-01-10T17:56:09.190ZUpstairs last five completed: 30 29 28 27 26
27122024-01-10T17:56:09.190ZWrote out file "/tmp/test_repair_verify.data"
27132024-01-10T17:56:09.190Z
27142024-01-10T17:56:09.190ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27152024-01-10T17:56:09.190ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27162024-01-10T17:56:09.191Z 0 000-019 22 24 24 42 48 48 F T T <---
27172024-01-10T17:56:09.191Z 1 020-039 22 24 24 42 48 48 F F F <---
27182024-01-10T17:56:09.191Z 2 040-059 20 20 20 39 39 39 F F F
27192024-01-10T17:56:09.191Z 3 060-079 20 24 24 39 48 48 F F F <---
27202024-01-10T17:56:09.192Z 4 080-099 24 24 24 46 48 48 F F F <---
27212024-01-10T17:56:09.193Z 5 100-119 24 24 24 46 49 49 T F F <---
27222024-01-10T17:56:09.193Z 6 120-139 22 22 22 44 44 44 F F F
27232024-01-10T17:56:09.194Z 7 140-159 14 14 14 21 21 21 F F F
27242024-01-10T17:56:09.195Z 8 160-179 23 23 23 45 45 45 F F F
27252024-01-10T17:56:09.196Z 9 180-199 23 23 23 45 45 45 F F F
27262024-01-10T17:56:09.196Z 10 200-219 18 24 24 29 49 49 F F F <---
27272024-01-10T17:56:09.196Z 11 220-239 20 20 20 39 39 39 F F F
27282024-01-10T17:56:09.215Z 12 240-259 16 16 16 27 27 27 F F F
27292024-01-10T17:56:09.216Z 13 260-279 19 19 19 33 33 33 F F F
27302024-01-10T17:56:09.216Z 14 280-299 22 22 22 41 41 41 F F F
27312024-01-10T17:56:09.216Z 15 300-319 22 22 22 41 41 41 F F F
27322024-01-10T17:56:09.216Z 16 320-339 23 23 23 45 45 45 F T T <---
27332024-01-10T17:56:09.216Z 17 340-359 12 12 12 19 19 19 F F F
27342024-01-10T17:56:09.216Z 18 360-379 22 22 22 42 42 42 F F F
27352024-01-10T17:56:09.216Z 19 380-399 19 19 19 33 33 33 F T T <---
27362024-01-10T17:56:09.216Z 20 400-419 23 23 23 45 45 45 F T T <---
27372024-01-10T17:56:09.216Z 21 420-439 19 19 19 33 33 33 F F F
27382024-01-10T17:56:09.216Z 22 440-459 18 18 18 32 32 32 F F F
27392024-01-10T17:56:09.216Z 23 460-479 19 19 19 33 33 33 F T T <---
27402024-01-10T17:56:09.216Z 24 480-499 20 20 20 39 39 39 F F F
27412024-01-10T17:56:09.217Z 25 500-519 18 24 24 30 49 49 F F F <---
27422024-01-10T17:56:09.217Z 26 520-539 11 11 11 18 18 18 F F F
27432024-01-10T17:56:09.217Z 27 540-559 20 20 20 39 39 39 F F F
27442024-01-10T17:56:09.217Z 28 560-579 22 24 24 42 47 47 F F F <---
27452024-01-10T17:56:09.217Z 29 580-599 22 22 22 42 42 42 F F F
27462024-01-10T17:56:09.217ZMax gen: 24, Max flush: 49
27472024-01-10T17:56:09.217ZError: Difference in extent metadata found!
27482024-01-10T17:56:09.217ZA Difference in extent metadata is expected here
27492024-01-10T17:56:09.217ZOn loop 12
27502024-01-10T17:56:09.219Z
27512024-01-10T17:56:09.219ZVerifying data now
27522024-01-10T17:56:14.125ZNo change after verify
27532024-01-10T17:56:14.125ZLoop: 12 Downstairs dump after verify (and repair):
27542024-01-10T17:56:14.147ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27552024-01-10T17:56:14.147Z 0 000-019 25 25 25 50 50 50 F F F
27562024-01-10T17:56:14.148Z 1 020-039 24 24 24 48 48 48 F F F
27572024-01-10T17:56:14.148Z 2 040-059 20 20 20 39 39 39 F F F
27582024-01-10T17:56:14.148Z 3 060-079 24 24 24 48 48 48 F F F
27592024-01-10T17:56:14.148Z 4 080-099 24 24 24 48 48 48 F F F
27602024-01-10T17:56:14.148Z 5 100-119 24 24 24 49 49 49 F F F
27612024-01-10T17:56:14.148Z 6 120-139 22 22 22 44 44 44 F F F
27622024-01-10T17:56:14.172Z 7 140-159 14 14 14 21 21 21 F F F
27632024-01-10T17:56:14.172Z 8 160-179 23 23 23 45 45 45 F F F
27642024-01-10T17:56:14.172Z 9 180-199 23 23 23 45 45 45 F F F
27652024-01-10T17:56:14.172Z 10 200-219 24 24 24 49 49 49 F F F
27662024-01-10T17:56:14.172Z 11 220-239 20 20 20 39 39 39 F F F
27672024-01-10T17:56:14.172Z 12 240-259 16 16 16 27 27 27 F F F
27682024-01-10T17:56:14.173Z 13 260-279 19 19 19 33 33 33 F F F
27692024-01-10T17:56:14.173Z 14 280-299 22 22 22 41 41 41 F F F
27702024-01-10T17:56:14.173Z 15 300-319 22 22 22 41 41 41 F F F
27712024-01-10T17:56:14.173Z 16 320-339 25 25 25 50 50 50 F F F
27722024-01-10T17:56:14.173Z 17 340-359 12 12 12 19 19 19 F F F
27732024-01-10T17:56:14.173Z 18 360-379 22 22 22 42 42 42 F F F
27742024-01-10T17:56:14.173Z 19 380-399 25 25 25 50 50 50 F F F
27752024-01-10T17:56:14.173Z 20 400-419 25 25 25 50 50 50 F F F
27762024-01-10T17:56:14.173Z 21 420-439 19 19 19 33 33 33 F F F
27772024-01-10T17:56:14.173Z 22 440-459 18 18 18 32 32 32 F F F
27782024-01-10T17:56:14.173Z 23 460-479 25 25 25 50 50 50 F F F
27792024-01-10T17:56:14.173Z 24 480-499 20 20 20 39 39 39 F F F
27802024-01-10T17:56:14.174Z 25 500-519 24 24 24 49 49 49 F F F
27812024-01-10T17:56:14.174Z 26 520-539 11 11 11 18 18 18 F F F
27822024-01-10T17:56:14.174Z 27 540-559 20 20 20 39 39 39 F F F
27832024-01-10T17:56:14.174Z 28 560-579 24 24 24 47 47 47 F F F
27842024-01-10T17:56:14.174Z 29 580-599 22 22 22 42 42 42 F F F
27852024-01-10T17:56:14.174ZMax gen: 25, Max flush: 50
27862024-01-10T17:56:14.174Z
27872024-01-10T17:56:14.174ZBegin loop 13
27882024-01-10T17:56:14.174ZDownstairs to restart: 0
27892024-01-10T17:56:14.185Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.996688838Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259}
27902024-01-10T17:56:14.185Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.99721211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259}
27912024-01-10T17:56:14.193Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997286207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259}
27922024-01-10T17:56:14.193Z{"msg":"Crucible 33795fc0-84a9-4f7c-ba6c-c94248c0b98d has session id: 09543eb0-7d29-4af5-a025-8e7aa41915bb","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997367937Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
27932024-01-10T17:56:14.193Z{"msg":"Upstairs opts: Upstairs UUID: 33795fc0-84a9-4f7c-ba6c-c94248c0b98d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":Crucible runtime is spawned
27942024-01-10T17:56:14.193Z0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997513641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
27952024-01-10T17:56:14.193Z{"msg":"Crucible stats registered with UUID: 33795fc0-84a9-4f7c-ba6c-c94248c0b98d","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997593378Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
27962024-01-10T17:56:14.193Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997649117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259}
27972024-01-10T17:56:14.193Z{"msg":"33795fc0-84a9-4f7c-ba6c-c94248c0b98d active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997685197Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
27982024-01-10T17:56:14.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997728044Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"0","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
27992024-01-10T17:56:14.194Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997764878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"0","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28002024-01-10T17:56:14.194Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997809312Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"1","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28012024-01-10T17:56:14.194Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997912775Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"1","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28022024-01-10T17:56:14.194Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.997957084Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"2","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28032024-01-10T17:56:14.194Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.998013422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"2","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28042024-01-10T17:56:14.194Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.998038823Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28052024-01-10T17:56:14.194Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.998062096Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28062024-01-10T17:56:14.195Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.998147227Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28072024-01-10T17:56:14.196Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.99835962Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28082024-01-10T17:56:14.196Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.998502827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28092024-01-10T17:56:14.196Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.998552656Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28102024-01-10T17:56:14.196Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.999681014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"2","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28112024-01-10T17:56:14.196Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.999750251Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"2","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28122024-01-10T17:56:14.196Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.999871879Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"1","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28132024-01-10T17:56:14.196Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:13.999924418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"1","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28142024-01-10T17:56:14.197Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.000477675Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"2","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28152024-01-10T17:56:14.197Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.000656913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"1","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28162024-01-10T17:56:14.197Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.001456052Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"2","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28172024-01-10T17:56:14.197Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.001608102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"2","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28182024-01-10T17:56:14.197Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.001653768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28192024-01-10T17:56:14.197Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.001710947Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"1","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28202024-01-10T17:56:14.197Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.001754391Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"1","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28212024-01-10T17:56:14.197Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.001816992Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28222024-01-10T17:56:14.197Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.001855604Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"0","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28232024-01-10T17:56:14.197Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.001918465Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"0","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28242024-01-10T17:56:14.216Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.002899546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"0","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28252024-01-10T17:56:14.216Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.003637342Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"0","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28262024-01-10T17:56:14.216Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.003706376Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"0","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28272024-01-10T17:56:14.216Z{"msg":"[0]R flush_numbers[0..12]: [50, 48, 39, 48, 48, 49, 44, 21, 45, 45, 49, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.00380798Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28282024-01-10T17:56:14.216Z{"msg":"[0]R generation[0..12]: [25, 24, 20, 24, 24, 24, 22, 14, 23, 23, 24, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.003852998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28292024-01-10T17:56:14.216Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.00388007Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28302024-01-10T17:56:14.216Z{"msg":"[1]R flush_numbers[0..12]: [50, 48, 39, 48, 48, 49, 44, 21, 45, 45, 49, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.003903861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28312024-01-10T17:56:14.216Z{"msg":"[1]R generation[0..12]: [25, 24, 20, 24, 24, 24, 22, 14, 23, 23, 24, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.003926963Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28322024-01-10T17:56:14.216Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
28332024-01-10T17:56:14.216Z,"time":"2024-01-10T17:56:14.003950896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28342024-01-10T17:56:14.217Z{"msg":"[2]R flush_numbers[0..12]: [50, 48, 39, 48, 48, 49, 44, 21, 45, 45, 49, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.003977074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28352024-01-10T17:56:14.217Z{"msg":"[2]R generation[0..12]: [25, 24, 20, 24, 24, 24, 22, 14, 23, 23, 24, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004000205Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28362024-01-10T17:56:14.217Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004024412Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28372024-01-10T17:56:14.217Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004056238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28382024-01-10T17:56:14.217Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004079943Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28392024-01-10T17:56:14.217Z{"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004100751Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28402024-01-10T17:56:14.217Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004122116Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"":"downstairs","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28412024-01-10T17:56:14.217Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004143522Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28422024-01-10T17:56:14.217Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004165906Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"0","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28432024-01-10T17:56:14.217Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004212625Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"1","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28442024-01-10T17:56:14.217Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004238064Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"client":"2","":"downstairs","session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28452024-01-10T17:56:14.217Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004284479Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28462024-01-10T17:56:14.218Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004308123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28472024-01-10T17:56:14.218Z{"msg":"33795fc0-84a9-4f7c-ba6c-c94248c0b98d is now active with session: 09543eb0-7d29-4af5-a025-8e7aa41915bb","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004329631Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28482024-01-10T17:56:14.218Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.004348993Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259,"session_id":"09543eb0-7d29-4af5-a025-8e7aa41915bb"}
28492024-01-10T17:56:14.218Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:14.00436922Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1259}
28502024-01-10T17:56:14.218ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28512024-01-10T17:56:14.218ZLoading write count information from file "/tmp/test_repair_verify.data"
28522024-01-10T17:56:14.218ZRun Repair workload
28532024-01-10T17:56:14.218Z01/30 Write block 367 len 20480 data: 2 2 2 2 3
28542024-01-10T17:56:14.218Z02/30 Read block 18 len 36864
28552024-01-10T17:56:14.218Z03/30 Write block 563 len 32768 data: 3 4 5 4 4 5 5 4
28562024-01-10T17:56:14.218Z04/30 Read block 45 len 12288
28572024-01-10T17:56:14.218Z05/30 Read block 242 len 12288
28582024-01-10T17:56:14.218Z06/30 Read block 315 len 32768
28592024-01-10T17:56:14.218Z07/30 Write block 526 len 32768 data: 2 2 2 2 2 3 3 3
28602024-01-10T17:56:14.219Z08/30 Write block 83 len 4096 data: 4
28612024-01-10T17:56:14.219Z09/30 Write block 406 len 32768 data: 2 3 3 3 4 5 5 5
28622024-01-10T17:56:14.219Z10/30 Write block 289 len 16384 data: 3 4 5 3
28632024-01-10T17:56:14.219Z11/30 Read block 51 len 24576
28642024-01-10T17:56:14.219Z12/30 Write block 545 len 24576 data: 5 5 5 4 4 4
28652024-01-10T17:56:14.226Z13/30 Read block 282 len 8192
28662024-01-10T17:56:14.226Z14/30 Read block 460 len 20480
28672024-01-10T17:56:14.226Z15/30 Write block 64 len 28672 data: 3 4 5 3 3 3 3
28682024-01-10T17:56:14.226Z16/30 Flush
28692024-01-10T17:56:14.226Z17/30 Write block 95 len 8192 data: 3 3
28702024-01-10T17:56:14.226Z18/30 Read block 409 len 16384
28712024-01-10T17:56:14.245Z19/30 Write block 422 len 32768 data: 4 3 3 2 2 2 3 3
28722024-01-10T17:56:14.245Z20/30 Read block 219 len 20480
28732024-01-10T17:56:14.246Z21/30 Write block 182 len 16384 data: 3 3 3 3
28742024-01-10T17:56:14.246Z22/30 Read block 169 len 36864
28752024-01-10T17:56:14.246Z23/30 Write block 23 len 32768 data: 4 4 4 4 4 4 5 5
28762024-01-10T17:56:14.246Z24/30 Flush
28772024-01-10T17:56:14.246Z25/30 Flush
28782024-01-10T17:56:14.246Z26/30 Write block 314 len 12288 data: 2 2 2
28792024-01-10T17:56:14.246Z27/30 Write block 48 len 8192 data: 3 4
28802024-01-10T17:56:14.246Z28/30 Read block 177 len 4096
28812024-01-10T17:56:14.246Z29/30 Write block 316 len 40960 data: 3 2 2 3 3 3 4 3 5 5
28822024-01-10T17:56:14.246Z30/30 Read block 177 len 12288
28832024-01-10T17:56:14.265Z----------------------------------------------------------------
28842024-01-10T17:56:14.265Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
28852024-01-10T17:56:14.265ZStates: Active Active Active
28862024-01-10T17:56:14.265ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
28872024-01-10T17:56:14.265Z 1 Acked 1000 Write 5 Done Done Done false
28882024-01-10T17:56:14.265Z 2 Acked 1001 Read 9 Done Done Done false
28892024-01-10T17:56:14.265Z 3 Acked 1002 Write 8 Sent Done Done false
28902024-01-10T17:56:14.265Z 4 Acked 1003 Read 3 Sent Done Done false
28912024-01-10T17:56:14.265Z 5 Acked 1004 Read 3 Sent Done Done false
28922024-01-10T17:56:14.265Z 6 Acked 1005 Read 8 Sent Done Done false
28932024-01-10T17:56:14.265Z 7 Acked 1006 Write 8 Sent Done Done false
28942024-01-10T17:56:14.265Z 8 Acked 1007 Write 1 Sent Done Done false
28952024-01-10T17:56:14.265Z 9 Acked 1008 Write 8 Sent Done Done false
28962024-01-10T17:56:14.265Z 10 Acked 1009 Write 4 Sent Done Done false
28972024-01-10T17:56:14.265Z 11 Acked 1010 Read 6 Sent Done Done false
28982024-01-10T17:56:14.265Z 12 Acked 1011 Write 6 Sent Done Done false
28992024-01-10T17:56:14.265Z 13 Acked 1012 Read 2 Sent Done Done false
29002024-01-10T17:56:14.265Z 14 Acked 1013 Read 5 Sent Done Done false
29012024-01-10T17:56:14.265Z 15 Acked 1014 Write 7 Sent Done Done false
29022024-01-10T17:56:14.266Z 16 Acked 1015 Flush 0 Sent Done Done false
29032024-01-10T17:56:14.266Z 17 Acked 1016 Write 2 Sent Done Done false
29042024-01-10T17:56:14.266Z 18 Acked 1017 Read 4 Sent Done Done false
29052024-01-10T17:56:14.266Z 19 Acked 1018 Write 8 Sent Done Done false
29062024-01-10T17:56:14.266Z 20 Acked 1019 Read 5 Sent Done Done false
29072024-01-10T17:56:14.266Z 21 Acked 1020 Write 4 Sent Done Done false
29082024-01-10T17:56:14.266Z 22 Acked 1021 Read 9 Sent Done Done false
29092024-01-10T17:56:14.266Z 23 Acked 1022 Write 8 Sent Done Done false
29102024-01-10T17:56:14.267Z 24 Acked 1023 Flush 0 Sent Done Done false
29112024-01-10T17:56:14.267Z 25 Acked 1024 Flush 0 Sent Done Done false
29122024-01-10T17:56:14.267Z 26 Acked 1025 Write 3 Sent Done Done false
29132024-01-10T17:56:14.267Z 27 Acked 1026 Write 2 Sent Done Done false
29142024-01-10T17:56:14.267Z 28 Acked 1027 Read 1 Sent Done Done false
29152024-01-10T17:56:14.267Z 29 Acked 1028 Write 10 Sent Done Done false
29162024-01-10T17:56:14.267Z 30 Acked 1029 Read 3 Sent Done Sent false
29172024-01-10T17:56:14.267Z STATES DS:0 DS:1 DS:2 TOTAL
29182024-01-10T17:56:14.267Z New 0 0 0 0
29192024-01-10T17:56:14.268Z Sent 28 0 1 29
29202024-01-10T17:56:14.268Z Done 2 30 29 61
29212024-01-10T17:56:14.268Z Skipped 0 0 0 0
29222024-01-10T17:56:14.268Z Error 0 0 0 0
29232024-01-10T17:56:14.268ZLast Flush: 0 1024 1024
29242024-01-10T17:56:14.268ZDownstairs last five completed:
29252024-01-10T17:56:14.268ZUpstairs last five completed: 30 29 28 27 26
29262024-01-10T17:56:14.268ZWrote out file "/tmp/test_repair_verify.data"
29272024-01-10T17:56:14.268Z
29282024-01-10T17:56:14.268ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29292024-01-10T17:56:14.289ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29302024-01-10T17:56:14.290Z 0 000-019 25 25 25 50 50 50 F F F
29312024-01-10T17:56:14.290Z 1 020-039 24 26 26 48 52 52 F F F <---
29322024-01-10T17:56:14.290Z 2 040-059 20 20 20 39 39 39 F T T <---
29332024-01-10T17:56:14.290Z 3 060-079 24 26 26 48 51 51 F F F <---
29342024-01-10T17:56:14.290Z 4 080-099 24 26 26 48 52 52 F F F <---
29352024-01-10T17:56:14.290Z 5 100-119 24 24 24 49 49 49 F F F
29362024-01-10T17:56:14.290Z 6 120-139 22 22 22 44 44 44 F F F
29372024-01-10T17:56:14.290Z 7 140-159 14 14 14 21 21 21 F F F
29382024-01-10T17:56:14.291Z 8 160-179 23 23 23 45 45 45 F F F
29392024-01-10T17:56:14.291Z 9 180-199 23 26 26 45 52 52 F F F <---
29402024-01-10T17:56:14.291Z 10 200-219 24 24 24 49 49 49 F F F
29412024-01-10T17:56:14.291Z 11 220-239 20 20 20 39 39 39 F F F
29422024-01-10T17:56:14.291Z 12 240-259 16 16 16 27 27 27 F F F
29432024-01-10T17:56:14.291Z 13 260-279 19 19 19 33 33 33 F F F
29442024-01-10T17:56:14.291Z 14 280-299 22 26 26 41 51 51 F F F <---
29452024-01-10T17:56:14.291Z 15 300-319 22 22 22 41 41 41 F T T <---
29462024-01-10T17:56:14.291Z 16 320-339 25 25 25 50 50 50 F T T <---
29472024-01-10T17:56:14.291Z 17 340-359 12 12 12 19 19 19 F F F
29482024-01-10T17:56:14.291Z 18 360-379 22 26 26 42 51 51 T F F <---
29492024-01-10T17:56:14.291Z 19 380-399 25 25 25 50 50 50 F F F
29502024-01-10T17:56:14.291Z 20 400-419 25 26 26 50 51 51 F F F <---
29512024-01-10T17:56:14.291Z 21 420-439 19 26 26 33 52 52 F F F <---
29522024-01-10T17:56:14.291Z 22 440-459 18 18 18 32 32 32 F F F
29532024-01-10T17:56:14.291Z 23 460-479 25 25 25 50 50 50 F F F
29542024-01-10T17:56:14.291Z 24 480-499 20 20 20 39 39 39 F F F
29552024-01-10T17:56:14.291Z 25 500-519 24 24 24 49 49 49 F F F
29562024-01-10T17:56:14.292Z 26 520-539 11 26 26 18 51 51 F F F <---
29572024-01-10T17:56:14.292Z 27 540-559 20 26 26 39 51 51 F F F <---
29582024-01-10T17:56:14.292Z 28 560-579 24 26 26 47 51 51 F F F <---
29592024-01-10T17:56:14.292Z 29 580-599 22 22 22 42 42 42 F F F
29602024-01-10T17:56:14.292ZMax gen: 26, Max flush: 52
29612024-01-10T17:56:14.292ZError: Difference in extent metadata found!
29622024-01-10T17:56:14.311ZA Difference in extent metadata is expected here
29632024-01-10T17:56:14.313ZOn loop 13
29642024-01-10T17:56:14.313Z
29652024-01-10T17:56:14.313ZVerifying data now
29662024-01-10T17:56:19.723ZNo change after verify
29672024-01-10T17:56:19.724ZLoop: 13 Downstairs dump after verify (and repair):
29682024-01-10T17:56:19.732ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29692024-01-10T17:56:19.733Z 0 000-019 25 25 25 50 50 50 F F F
29702024-01-10T17:56:19.733Z 1 020-039 26 26 26 52 52 52 F F F
29712024-01-10T17:56:19.733Z 2 040-059 27 27 27 53 53 53 F F F
29722024-01-10T17:56:19.733Z 3 060-079 26 26 26 51 51 51 F F F
29732024-01-10T17:56:19.733Z 4 080-099 26 26 26 52 52 52 F F F
29742024-01-10T17:56:19.733Z 5 100-119 24 24 24 49 49 49 F F F
29752024-01-10T17:56:19.733Z 6 120-139 22 22 22 44 44 44 F F F
29762024-01-10T17:56:19.733Z 7 140-159 14 14 14 21 21 21 F F F
29772024-01-10T17:56:19.733Z 8 160-179 23 23 23 45 45 45 F F F
29782024-01-10T17:56:19.733Z 9 180-199 26 26 26 52 52 52 F F F
29792024-01-10T17:56:19.733Z 10 200-219 24 24 24 49 49 49 F F F
29802024-01-10T17:56:19.734Z 11 220-239 20 20 20 39 39 39 F F F
29812024-01-10T17:56:19.734Z 12 240-259 16 16 16 27 27 27 F F F
29822024-01-10T17:56:19.734Z 13 260-279 19 19 19 33 33 33 F F F
29832024-01-10T17:56:19.734Z 14 280-299 26 26 26 51 51 51 F F F
29842024-01-10T17:56:19.734Z 15 300-319 27 27 27 53 53 53 F F F
29852024-01-10T17:56:19.734Z 16 320-339 27 27 27 53 53 53 F F F
29862024-01-10T17:56:19.734Z 17 340-359 12 12 12 19 19 19 F F F
29872024-01-10T17:56:19.735Z 18 360-379 26 26 26 51 51 51 F F F
29882024-01-10T17:56:19.737Z 19 380-399 25 25 25 50 50 50 F F F
29892024-01-10T17:56:19.737Z 20 400-419 26 26 26 51 51 51 F F F
29902024-01-10T17:56:19.737Z 21 420-439 26 26 26 52 52 52 F F F
29912024-01-10T17:56:19.737Z 22 440-459 18 18 18 32 32 32 F F F
29922024-01-10T17:56:19.737Z 23 460-479 25 25 25 50 50 50 F F F
29932024-01-10T17:56:19.737Z 24 480-499 20 20 20 39 39 39 F F F
29942024-01-10T17:56:19.737Z 25 500-519 24 24 24 49 49 49 F F F
29952024-01-10T17:56:19.737Z 26 520-539 26 26 26 51 51 51 F F F
29962024-01-10T17:56:19.737Z 27 540-559 26 26 26 51 51 51 F F F
29972024-01-10T17:56:19.737Z 28 560-579 26 26 26 51 51 51 F F F
29982024-01-10T17:56:19.737Z 29 580-599 22 22 22 42 42 42 F F F
29992024-01-10T17:56:19.737ZMax gen: 27, Max flush: 53
30002024-01-10T17:56:19.738Z
30012024-01-10T17:56:19.738ZBegin loop 14
30022024-01-10T17:56:19.738ZDownstairs to restart: 2
30032024-01-10T17:56:19.738Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.531444743Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267}
30042024-01-10T17:56:19.738Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.531986362Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267}
30052024-01-10T17:56:19.738Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532041692Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267}
30062024-01-10T17:56:19.738Z{"msg":"Crucible 659c9c61-20f5-4081-9721-1a4e4c3c224b has session id: e4543be6-7bc6-4d8a-8207-09c11fbf80b3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532073665Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30072024-01-10T17:56:19.740Z{"msg":"Upstairs opts: Upstairs UUID: 659c9c61-20f5-4081-9721-1a4e4c3c224b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532157929Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30082024-01-10T17:56:19.740Z{"msg":"Crucible stats registered with UUID: 659c9c61-20f5-4081-9721-1a4e4c3c224b","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532202254Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30092024-01-10T17:56:19.740ZCrucible runtime is spawned
30102024-01-10T17:56:19.740Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532429006Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267}
30112024-01-10T17:56:19.740Z{"msg":"659c9c61-20f5-4081-9721-1a4e4c3c224b active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532640858Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30122024-01-10T17:56:19.740Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532719289Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"0","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30132024-01-10T17:56:19.740Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532774347Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"0","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30142024-01-10T17:56:19.741Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532798872Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"1","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30152024-01-10T17:56:19.741Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532821729Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"1","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30162024-01-10T17:56:19.741Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532879523Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"2","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30172024-01-10T17:56:19.742Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.532921896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"2","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30182024-01-10T17:56:19.742Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.533075653Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30192024-01-10T17:56:19.742Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.533156369Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30202024-01-10T17:56:19.742Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.533465448Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30212024-01-10T17:56:19.742Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.533557037Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30222024-01-10T17:56:19.742Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.533630281Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30232024-01-10T17:56:19.742Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.533782145Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30242024-01-10T17:56:19.742Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.534378388Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"0","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30252024-01-10T17:56:19.742Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.534487064Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"0","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30262024-01-10T17:56:19.743Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.534530696Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"1","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30272024-01-10T17:56:19.743Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.534570015Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"1","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30282024-01-10T17:56:19.744Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.535225082Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"2","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30292024-01-10T17:56:19.744Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.535333563Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"2","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30302024-01-10T17:56:19.744Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.535482447Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"0","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30312024-01-10T17:56:19.744Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.535552103Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"1","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30322024-01-10T17:56:19.744Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.536123731Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"2","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30332024-01-10T17:56:19.744Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.536436865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"1","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30342024-01-10T17:56:19.744Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.536545475Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"1","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30352024-01-10T17:56:19.744Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.536617028Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30362024-01-10T17:56:19.744Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.53665777Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"0","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30372024-01-10T17:56:19.744Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.536704553Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"0","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30382024-01-10T17:56:19.744Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.536748651Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30392024-01-10T17:56:19.745Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.536819854Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"2","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30402024-01-10T17:56:19.745Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.536856921Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"2","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30412024-01-10T17:56:19.745Z{"msg":"[0]R flush_numbers[0..12]: [50, 52, 53, 51, 52, 49, 44, 21, 45, 52, 49, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.536918456Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30422024-01-10T17:56:19.745Z{"msg":"[0]R generation[0..12]: [25, 26, 27, 26, 26, 24, 22, 14, 23, 26, 24, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.53694592Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30432024-01-10T17:56:19.745Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.536989345Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30442024-01-10T17:56:19.745Z{"msg":"[1]R flush_numbers[0..12]: [50, 52, 53, 51, 52, 49, 44, 21, 45, 52, 49, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537014155Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30452024-01-10T17:56:19.745Z{"msg":"[1]R generation[0..12]: [25, 26, 27, 26, 26, 24, 22, 14, 23, 26, 24, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537055208Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30462024-01-10T17:56:19.745Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537082011Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30472024-01-10T17:56:19.745Z{"msg":"[2]R flush_numbers[0..12]: [50, 52, 53, 51, 52, 49, 44, 21, 45, 52, 49, 39]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
30482024-01-10T17:56:19.745Z,"time":"2024-01-10T17:56:19.537142391Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30492024-01-10T17:56:19.745Z{"msg":"[2]R generation[0..12]: [25, 26, 27, 26, 26, 24, 22, 14, 23, 26, 24, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537172179Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30502024-01-10T17:56:19.746Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537215118Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30512024-01-10T17:56:19.746Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.5372384Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30522024-01-10T17:56:19.746Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537260546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30532024-01-10T17:56:19.746Z{"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537300211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30542024-01-10T17:56:19.746Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537322862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"":"downstairs","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30552024-01-10T17:56:19.746Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537362977Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30562024-01-10T17:56:19.746Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537383873Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"0","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30572024-01-10T17:56:19.747Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537424048Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"1","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30582024-01-10T17:56:19.747Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537447409Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"client":"2","":"downstairs","session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30592024-01-10T17:56:19.747Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537488749Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30602024-01-10T17:56:19.747Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537510491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30612024-01-10T17:56:19.747Z{"msg":"659c9c61-20f5-4081-9721-1a4e4c3c224b is now active with session: e4543be6-7bc6-4d8a-8207-09c11fbf80b3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537533444Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30622024-01-10T17:56:19.747Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537552833Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267,"session_id":"e4543be6-7bc6-4d8a-8207-09c11fbf80b3"}
30632024-01-10T17:56:19.747Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:19.537572653Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1267}
30642024-01-10T17:56:19.747ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30652024-01-10T17:56:19.748ZLoading write count information from file "/tmp/test_repair_verify.data"
30662024-01-10T17:56:19.748ZRun Repair workload
30672024-01-10T17:56:19.748Z01/30 Write block 130 len 4096 data: 2
30682024-01-10T17:56:19.766Z02/30 Write block 501 len 28672 data: 4 4 5 4 4 4 5
30692024-01-10T17:56:19.766Z03/30 Write block 490 len 24576 data: 5 5 6 6 7 8
30702024-01-10T17:56:19.767Z04/30 Write block 198 len 28672 data: 4 5 4 4 4 3 2
30712024-01-10T17:56:19.767Z05/30 Write block 111 len 12288 data: 4 5 4
30722024-01-10T17:56:19.768Z06/30 Read block 448 len 12288
30732024-01-10T17:56:19.768Z07/30 Write block 324 len 4096 data: 6
30742024-01-10T17:56:19.768Z08/30 Write block 495 len 20480 data: 9 7 6 5 6
30752024-01-10T17:56:19.768Z09/30 Read block 551 len 16384
30762024-01-10T17:56:19.768Z10/30 Write block 385 len 4096 data: 2
30772024-01-10T17:56:19.768Z11/30 Read block 247 len 8192
30782024-01-10T17:56:19.768Z12/30 Write block 114 len 4096 data: 4
30792024-01-10T17:56:19.768Z13/30 Read block 539 len 16384
30802024-01-10T17:56:19.768Z14/30 Write block 180 len 28672 data: 3 3 4 4 4 4 3
30812024-01-10T17:56:19.768Z15/30 Write block 177 len 20480 data: 4 4 3 4 4
30822024-01-10T17:56:19.769Z16/30 Write block 574 len 40960 data: 3 3 4 4 3 4 4 4 5 5
30832024-01-10T17:56:19.769Z17/30 Read block 91 len 24576
30842024-01-10T17:56:19.769Z18/30 Read block 206 len 16384
30852024-01-10T17:56:19.769Z19/30 Write block 89 len 16384 data: 2 2 4 3
30862024-01-10T17:56:19.769Z20/30 Read block 215 len 16384
30872024-01-10T17:56:19.769Z21/30 Write block 484 len 12288 data: 3 3 4
30882024-01-10T17:56:19.769Z22/30 Read block 515 len 28672
30892024-01-10T17:56:19.769Z23/30 Read block 516 len 40960
30902024-01-10T17:56:19.769Z24/30 Write block 557 len 28672 data: 3 2 2 2 3 3 4
30912024-01-10T17:56:19.769Z25/30 Read block 549 len 36864
30922024-01-10T17:56:19.775Z26/30 Write block 441 len 4096 data: 6
30932024-01-10T17:56:19.775Z27/30 Write block 49 len 8192 data: 5 4
30942024-01-10T17:56:19.776Z28/30 Write block 537 len 20480 data: 4 4 3 3 3
30952024-01-10T17:56:19.776Z29/30 Read block 288 len 36864
30962024-01-10T17:56:19.776Z30/30 Write block 476 len 8192 data: 3 3
30972024-01-10T17:56:19.776Z----------------------------------------------------------------
30982024-01-10T17:56:19.777Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
30992024-01-10T17:56:19.777ZStates: Active Active Active
31002024-01-10T17:56:19.777ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31012024-01-10T17:56:19.777Z 1 Acked 1000 Write 1 Done Done Done false
31022024-01-10T17:56:19.777Z 2 Acked 1001 Write 7 Done Done Done false
31032024-01-10T17:56:19.777Z 3 Acked 1002 Write 6 Done Done Sent false
31042024-01-10T17:56:19.777Z 4 Acked 1003 Write 7 Done Done Sent false
31052024-01-10T17:56:19.777Z 5 Acked 1004 Write 3 Done Done Sent false
31062024-01-10T17:56:19.777Z 6 Acked 1005 Read 3 Done Done Sent false
31072024-01-10T17:56:19.777Z 7 Acked 1006 Write 1 Done Done Sent false
31082024-01-10T17:56:19.777Z 8 Acked 1007 Write 5 Done Done Sent false
31092024-01-10T17:56:19.777Z 9 Acked 1008 Read 4 Done Done Sent false
31102024-01-10T17:56:19.778Z 10 Acked 1009 Write 1 Done Done Sent false
31112024-01-10T17:56:19.778Z 11 Acked 1010 Read 2 Done Done Sent false
31122024-01-10T17:56:19.778Z 12 Acked 1011 Write 1 Done Done Sent false
31132024-01-10T17:56:19.778Z 13 Acked 1012 Read 4 Done Done Sent false
31142024-01-10T17:56:19.780Z 14 Acked 1013 Write 7 Done Done Sent false
31152024-01-10T17:56:19.780Z 15 Acked 1014 Write 5 Done Done Sent false
31162024-01-10T17:56:19.780Z 16 Acked 1015 Write 10 Done Done Sent false
31172024-01-10T17:56:19.780Z 17 Acked 1016 Read 6 Done Done Sent false
31182024-01-10T17:56:19.780Z 18 Acked 1017 Read 4 Done Done Sent false
31192024-01-10T17:56:19.780Z 19 Acked 1018 Write 4 Done Done Sent false
31202024-01-10T17:56:19.781Z 20 Acked 1019 Read 4 Done Done Sent false
31212024-01-10T17:56:19.781Z 21 Acked 1020 Write 3 Done Done Sent false
31222024-01-10T17:56:19.781Z 22 Acked 1021 Read 7 Done Done Sent false
31232024-01-10T17:56:19.781Z 23 Acked 1022 Read 10 Done Done Sent false
31242024-01-10T17:56:19.781Z 24 Acked 1023 Write 7 Done Done Sent false
31252024-01-10T17:56:19.781Z 25 Acked 1024 Read 9 Done Done Sent false
31262024-01-10T17:56:19.781Z 26 Acked 1025 Write 1 Done Done Sent false
31272024-01-10T17:56:19.781Z 27 Acked 1026 Write 2 Done Done Sent false
31282024-01-10T17:56:19.781Z 28 Acked 1027 Write 5 Done Done Sent false
31292024-01-10T17:56:19.781Z 29 Acked 1028 Read 9 Done Done Sent false
31302024-01-10T17:56:19.781Z 30 Acked 1029 Write 2 Sent Sent Sent false
31312024-01-10T17:56:19.782Z STATES DS:0 DS:1 DS:2 TOTAL
31322024-01-10T17:56:19.782Z New 0 0 0 0
31332024-01-10T17:56:19.782Z Sent 1 1 28 30
31342024-01-10T17:56:19.782Z Done 29 29 2 60
31352024-01-10T17:56:19.782Z Skipped 0 0 0 0
31362024-01-10T17:56:19.782Z Error 0 0 0 0
31372024-01-10T17:56:19.782ZLast Flush: 0 0 0
31382024-01-10T17:56:19.782ZDownstairs last five completed:
31392024-01-10T17:56:19.782ZUpstairs last five completed: 30 29 28 27 26
31402024-01-10T17:56:19.786ZWrote out file "/tmp/test_repair_verify.data"
31412024-01-10T17:56:19.786Z
31422024-01-10T17:56:19.790ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31432024-01-10T17:56:19.819ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31442024-01-10T17:56:19.819Z 0 000-019 25 25 25 50 50 50 F F F
31452024-01-10T17:56:19.820Z 1 020-039 26 26 26 52 52 52 F F F
31462024-01-10T17:56:19.820Z 2 040-059 27 27 27 53 53 53 T T F <---
31472024-01-10T17:56:19.820Z 3 060-079 26 26 26 51 51 51 F F F
31482024-01-10T17:56:19.820Z 4 080-099 26 26 26 52 52 52 T T F <---
31492024-01-10T17:56:19.824Z 5 100-119 24 24 24 49 49 49 T T F <---
31502024-01-10T17:56:19.824Z 6 120-139 22 22 22 44 44 44 T T T
31512024-01-10T17:56:19.824Z 7 140-159 14 14 14 21 21 21 F F F
31522024-01-10T17:56:19.824Z 8 160-179 23 23 23 45 45 45 T T F <---
31532024-01-10T17:56:19.824Z 9 180-199 26 26 26 52 52 52 T T F <---
31542024-01-10T17:56:19.825Z 10 200-219 24 24 24 49 49 49 T T F <---
31552024-01-10T17:56:19.825Z 11 220-239 20 20 20 39 39 39 F F F
31562024-01-10T17:56:19.825Z 12 240-259 16 16 16 27 27 27 F F F
31572024-01-10T17:56:19.825Z 13 260-279 19 19 19 33 33 33 F F F
31582024-01-10T17:56:19.825Z 14 280-299 26 26 26 51 51 51 F F F
31592024-01-10T17:56:19.825Z 15 300-319 27 27 27 53 53 53 F F F
31602024-01-10T17:56:19.825Z 16 320-339 27 27 27 53 53 53 T T F <---
31612024-01-10T17:56:19.825Z 17 340-359 12 12 12 19 19 19 F F F
31622024-01-10T17:56:19.825Z 18 360-379 26 26 26 51 51 51 F F F
31632024-01-10T17:56:19.825Z 19 380-399 25 25 25 50 50 50 T T F <---
31642024-01-10T17:56:19.825Z 20 400-419 26 26 26 51 51 51 F F F
31652024-01-10T17:56:19.826Z 21 420-439 26 26 26 52 52 52 F F F
31662024-01-10T17:56:19.826Z 22 440-459 18 18 18 32 32 32 T T F <---
31672024-01-10T17:56:19.826Z 23 460-479 25 25 25 50 50 50 T T F <---
31682024-01-10T17:56:19.826Z 24 480-499 20 20 20 39 39 39 T T F <---
31692024-01-10T17:56:19.827Z 25 500-519 24 24 24 49 49 49 T T T
31702024-01-10T17:56:19.827Z 26 520-539 26 26 26 51 51 51 T T F <---
31712024-01-10T17:56:19.827Z 27 540-559 26 26 26 51 51 51 T T F <---
31722024-01-10T17:56:19.827Z 28 560-579 26 26 26 51 51 51 T T F <---
31732024-01-10T17:56:19.827Z 29 580-599 22 22 22 42 42 42 T T F <---
31742024-01-10T17:56:19.827ZMax gen: 27, Max flush: 53
31752024-01-10T17:56:19.827ZError: Difference in extent metadata found!
31762024-01-10T17:56:19.831ZA Difference in extent metadata is expected here
31772024-01-10T17:56:19.831ZOn loop 14
31782024-01-10T17:56:19.831Z
31792024-01-10T17:56:19.831ZVerifying data now
31802024-01-10T17:56:25.430ZNo change after verify
31812024-01-10T17:56:25.430ZLoop: 14 Downstairs dump after verify (and repair):
31822024-01-10T17:56:25.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31832024-01-10T17:56:25.453Z 0 000-019 25 25 25 50 50 50 F F F
31842024-01-10T17:56:25.457Z 1 020-039 26 26 26 52 52 52 F F F
31852024-01-10T17:56:25.457Z 2 040-059 28 28 28 54 54 54 F F F
31862024-01-10T17:56:25.459Z 3 060-079 26 26 26 51 51 51 F F F
31872024-01-10T17:56:25.459Z 4 080-099 28 28 28 54 54 54 F F F
31882024-01-10T17:56:25.459Z 5 100-119 28 28 28 54 54 54 F F F
31892024-01-10T17:56:25.459Z 6 120-139 28 28 28 54 54 54 F F F
31902024-01-10T17:56:25.459Z 7 140-159 14 14 14 21 21 21 F F F
31912024-01-10T17:56:25.459Z 8 160-179 28 28 28 54 54 54 F F F
31922024-01-10T17:56:25.459Z 9 180-199 28 28 28 54 54 54 F F F
31932024-01-10T17:56:25.459Z 10 200-219 28 28 28 54 54 54 F F F
31942024-01-10T17:56:25.459Z 11 220-239 20 20 20 39 39 39 F F F
31952024-01-10T17:56:25.459Z 12 240-259 16 16 16 27 27 27 F F F
31962024-01-10T17:56:25.459Z 13 260-279 19 19 19 33 33 33 F F F
31972024-01-10T17:56:25.459Z 14 280-299 26 26 26 51 51 51 F F F
31982024-01-10T17:56:25.459Z 15 300-319 27 27 27 53 53 53 F F F
31992024-01-10T17:56:25.460Z 16 320-339 28 28 28 54 54 54 F F F
32002024-01-10T17:56:25.461Z 17 340-359 12 12 12 19 19 19 F F F
32012024-01-10T17:56:25.461Z 18 360-379 26 26 26 51 51 51 F F F
32022024-01-10T17:56:25.461Z 19 380-399 28 28 28 54 54 54 F F F
32032024-01-10T17:56:25.461Z 20 400-419 26 26 26 51 51 51 F F F
32042024-01-10T17:56:25.461Z 21 420-439 26 26 26 52 52 52 F F F
32052024-01-10T17:56:25.461Z 22 440-459 28 28 28 54 54 54 F F F
32062024-01-10T17:56:25.461Z 23 460-479 28 28 28 54 54 54 F F F
32072024-01-10T17:56:25.461Z 24 480-499 28 28 28 54 54 54 F F F
32082024-01-10T17:56:25.461Z 25 500-519 28 28 28 54 54 54 F F F
32092024-01-10T17:56:25.461Z 26 520-539 28 28 28 54 54 54 F F F
32102024-01-10T17:56:25.461Z 27 540-559 28 28 28 54 54 54 F F F
32112024-01-10T17:56:25.461Z 28 560-579 28 28 28 54 54 54 F F F
32122024-01-10T17:56:25.461Z 29 580-599 28 28 28 54 54 54 F F F
32132024-01-10T17:56:25.461ZMax gen: 28, Max flush: 54
32142024-01-10T17:56:25.461Z
32152024-01-10T17:56:25.461ZBegin loop 15
32162024-01-10T17:56:25.461ZDownstairs to restart: 0
32172024-01-10T17:56:25.485Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.298954446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275}
32182024-01-10T17:56:25.490Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.29942417Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275}
32192024-01-10T17:56:25.490Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.299481236Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275}
32202024-01-10T17:56:25.491Z{"msg":"Crucible 53095400-5456-4dce-9367-56f3550dcde7 has session id: 524c35c9-fe1f-4995-b977-bee2953edfe7","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.299504512Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32212024-01-10T17:56:25.491Z{"msg":"Upstairs opts: Upstairs UUID: 53095400-5456-4dce-9367-56f3550dcde7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.299610722Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32222024-01-10T17:56:25.491Z{"msg":"Crucible stats registered with UUID: 53095400-5456-4dce-9367-56f3550dcde7","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.299655955Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32232024-01-10T17:56:25.491ZCrucible runtime is spawned
32242024-01-10T17:56:25.491Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.299974993Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275}
32252024-01-10T17:56:25.491Z{"msg":"53095400-5456-4dce-9367-56f3550dcde7 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.30005873Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32262024-01-10T17:56:25.491Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300102895Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"0","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32272024-01-10T17:56:25.491Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300147014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"0","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32282024-01-10T17:56:25.491Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300186052Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"1","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32292024-01-10T17:56:25.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300306977Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"1","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32302024-01-10T17:56:25.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300474225Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"2","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32312024-01-10T17:56:25.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300521597Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"2","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32322024-01-10T17:56:25.492Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300561171Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32332024-01-10T17:56:25.492Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300697206Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32342024-01-10T17:56:25.492Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300783159Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32352024-01-10T17:56:25.492Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300834141Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32362024-01-10T17:56:25.492Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.300975228Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32372024-01-10T17:56:25.492Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.301022631Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32382024-01-10T17:56:25.492Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.302045775Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"1","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32392024-01-10T17:56:25.493Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.302115095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"1","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32402024-01-10T17:56:25.493Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.302141246Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"0","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32412024-01-10T17:56:25.493Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.302163831Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"0","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32422024-01-10T17:56:25.493Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.302220256Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"2","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32432024-01-10T17:56:25.493Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.302247377Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"2","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32442024-01-10T17:56:25.498Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.303081581Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"2","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32452024-01-10T17:56:25.498Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.303227752Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"1","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32462024-01-10T17:56:25.499Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.303297161Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"0","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32472024-01-10T17:56:25.499Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304166058Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"2","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32482024-01-10T17:56:25.499Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304277442Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"2","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32492024-01-10T17:56:25.499Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304350824Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32502024-01-10T17:56:25.499Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304393548Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"0","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32512024-01-10T17:56:25.499Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304464169Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"0","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32522024-01-10T17:56:25.499Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304533929Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32532024-01-10T17:56:25.499Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304600296Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"1","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32542024-01-10T17:56:25.499Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304669956Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"1","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32552024-01-10T17:56:25.500Z{"msg":"[0]R flush_numbers[0..12]: [50, 52, 54, 51, 54, 54, 54, 21, 54, 54, 54, 39]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
32562024-01-10T17:56:25.500Z,"time":"2024-01-10T17:56:25.304739258Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32572024-01-10T17:56:25.500Z{"msg":"[0]R generation[0..12]: [25, 26, 28, 26, 28, 28, 28, 14, 28, 28, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304818504Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32582024-01-10T17:56:25.500Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304891946Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32592024-01-10T17:56:25.500Z{"msg":"[1]R flush_numbers[0..12]: [50, 52, 54, 51, 54, 54, 54, 21, 54, 54, 54, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.304965242Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32602024-01-10T17:56:25.500Z{"msg":"[1]R generation[0..12]: [25, 26, 28, 26, 28, 28, 28, 14, 28, 28, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305011984Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32612024-01-10T17:56:25.500Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305087436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32622024-01-10T17:56:25.500Z{"msg":"[2]R flush_numbers[0..12]: [50, 52, 54, 51, 54, 54, 54, 21, 54, 54, 54, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305135471Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32632024-01-10T17:56:25.500Z{"msg":"[2]R generation[0..12]: [25, 26, 28, 26, 28, 28, 28, 14, 28, 28, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.30521324Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32642024-01-10T17:56:25.500Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305263059Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32652024-01-10T17:56:25.500Z{"msg":"Max found gen is 29","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305332027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32662024-01-10T17:56:25.500Z{"msg":"Generation requested: 30 >= found:29","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305398807Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32672024-01-10T17:56:25.500Z{"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305428651Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32682024-01-10T17:56:25.500ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
32692024-01-10T17:56:25.500Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.30549675Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"":"downstairs","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32702024-01-10T17:56:25.500Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305565857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32712024-01-10T17:56:25.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305630168Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"0","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32722024-01-10T17:56:25.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305700608Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"1","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32732024-01-10T17:56:25.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.30576967Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"client":"2","":"downstairs","session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32742024-01-10T17:56:25.500Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305810511Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32752024-01-10T17:56:25.500Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.305873498Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32762024-01-10T17:56:25.500Z{"msg":"53095400-5456-4dce-9367-56f3550dcde7 is now active with session: 524c35c9-fe1f-4995-b977-bee2953edfe7","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.30593862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32772024-01-10T17:56:25.500Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.306001663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275,"session_id":"524c35c9-fe1f-4995-b977-bee2953edfe7"}
32782024-01-10T17:56:25.500Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:25.306066813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1275}
32792024-01-10T17:56:25.502ZLoading write count information from file "/tmp/test_repair_verify.data"
32802024-01-10T17:56:25.502ZRun Repair workload
32812024-01-10T17:56:25.502Z01/30 Flush
32822024-01-10T17:56:25.502Z02/30 Write block 30 len 16384 data: 6 4 4 3
32832024-01-10T17:56:25.502Z03/30 Read block 57 len 32768
32842024-01-10T17:56:25.506Z04/30 Write block 391 len 16384 data: 2 2 2 3
32852024-01-10T17:56:25.506Z05/30 Flush
32862024-01-10T17:56:25.506Z06/30 Write block 134 len 8192 data: 2 2
32872024-01-10T17:56:25.506Z07/30 Write block 75 len 20480 data: 2 2 2 2 4
32882024-01-10T17:56:25.506Z08/30 Write block 593 len 20480 data: 5 5 4 4 3
32892024-01-10T17:56:25.506Z09/30 Read block 251 len 16384
32902024-01-10T17:56:25.509Z10/30 Write block 433 len 8192 data: 3 3
32912024-01-10T17:56:25.509Z11/30 Read block 598 len 4096
32922024-01-10T17:56:25.509Z12/30 Flush
32932024-01-10T17:56:25.512Z13/30 Write block 495 len 36864 data: 10 8 7 6 7 4 5 5 6
32942024-01-10T17:56:25.515Z14/30 Read block 65 len 28672
32952024-01-10T17:56:25.518Z15/30 Read block 33 len 32768
32962024-01-10T17:56:25.521Z16/30 Read block 349 len 24576
32972024-01-10T17:56:25.524Z17/30 Read block 517 len 8192
32982024-01-10T17:56:25.524Z18/30 Read block 67 len 40960
32992024-01-10T17:56:25.527Z19/30 Read block 490 len 40960
33002024-01-10T17:56:25.530Z20/30 Write block 413 len 12288 data: 6 5 3
33012024-01-10T17:56:25.533Z21/30 Read block 579 len 4096
33022024-01-10T17:56:25.533Z22/30 Read block 184 len 16384
33032024-01-10T17:56:25.536Z23/30 Read block 314 len 36864
33042024-01-10T17:56:25.537Z24/30 Flush
33052024-01-10T17:56:25.540Z25/30 Write block 303 len 4096 data: 3
33062024-01-10T17:56:25.540Z26/30 Read block 530 len 20480
33072024-01-10T17:56:25.543Z27/30 Read block 187 len 28672
33082024-01-10T17:56:25.546Z28/30 Read block 361 len 20480
33092024-01-10T17:56:25.550Z29/30 Write block 53 len 8192 data: 4 5
33102024-01-10T17:56:25.550Z30/30 Write block 91 len 36864 data: 5 4 3 3 4 4 4 4 4
33112024-01-10T17:56:25.550Z----------------------------------------------------------------
33122024-01-10T17:56:25.550Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:29
33132024-01-10T17:56:25.550ZStates: Active Active Active
33142024-01-10T17:56:25.550ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33152024-01-10T17:56:25.550Z 2 Acked 1001 Write 4 Done Done Done false
33162024-01-10T17:56:25.550Z 3 Acked 1002 Read 8 Done Done Done false
33172024-01-10T17:56:25.550Z 4 Acked 1003 Write 4 Sent Done Done false
33182024-01-10T17:56:25.550Z 5 Acked 1004 Flush 0 Sent Done Done false
33192024-01-10T17:56:25.550Z 6 Acked 1005 Write 2 Sent Done Done false
33202024-01-10T17:56:25.550Z 7 Acked 1006 Write 5 Sent Done Done false
33212024-01-10T17:56:25.550Z 8 Acked 1007 Write 5 Sent Done Done false
33222024-01-10T17:56:25.550Z 9 Acked 1008 Read 4 Sent Done Done false
33232024-01-10T17:56:25.550Z 10 Acked 1009 Write 2 Sent Done Done false
33242024-01-10T17:56:25.550Z 11 Acked 1010 Read 1 Sent Done Done false
33252024-01-10T17:56:25.550Z 12 Acked 1011 Flush 0 Sent Done Done false
33262024-01-10T17:56:25.552Z 13 Acked 1012 Write 9 Sent Done Done false
33272024-01-10T17:56:25.552Z 14 Acked 1013 Read 7 Sent Done Done false
33282024-01-10T17:56:25.552Z 15 Acked 1014 Read 8 Sent Done Done false
33292024-01-10T17:56:25.552Z 16 Acked 1015 Read 6 Sent Done Done false
33302024-01-10T17:56:25.552Z 17 Acked 1016 Read 2 Sent Done Done false
33312024-01-10T17:56:25.552Z 18 Acked 1017 Read 10 Sent Done Done false
33322024-01-10T17:56:25.552Z 19 Acked 1018 Read 10 Sent Done Done false
33332024-01-10T17:56:25.552Z 20 Acked 1019 Write 3 Sent Done Done false
33342024-01-10T17:56:25.552Z 21 Acked 1020 Read 1 Sent Done Done false
33352024-01-10T17:56:25.552Z 22 Acked 1021 Read 4 Sent Done Done false
33362024-01-10T17:56:25.552Z 23 Acked 1022 Read 9 Sent Done Done false
33372024-01-10T17:56:25.552Z 24 Acked 1023 Flush 0 Sent Done Done false
33382024-01-10T17:56:25.552Z 25 Acked 1024 Write 1 Sent Done Done false
33392024-01-10T17:56:25.552Z 26 Acked 1025 Read 5 Sent Done Done false
33402024-01-10T17:56:25.552Z 27 Acked 1026 Read 7 Sent Done Done false
33412024-01-10T17:56:25.552Z 28 Acked 1027 Read 5 Sent Done Done false
33422024-01-10T17:56:25.552Z 29 Acked 1028 Write 2 Sent Done Done false
33432024-01-10T17:56:25.552Z 30 Acked 1029 Write 9 Sent Sent Sent false
33442024-01-10T17:56:25.553Z STATES DS:0 DS:1 DS:2 TOTAL
33452024-01-10T17:56:25.553Z New 0 0 0 0
33462024-01-10T17:56:25.553Z Sent 27 1 1 29
33472024-01-10T17:56:25.553Z Done 2 28 28 58
33482024-01-10T17:56:25.553Z Skipped 0 0 0 0
33492024-01-10T17:56:25.553Z Error 0 0 0 0
33502024-01-10T17:56:25.553ZLast Flush: 1000 1023 1023
33512024-01-10T17:56:25.553ZDownstairs last five completed: 1000
33522024-01-10T17:56:25.553ZUpstairs last five completed: 30 29 28 27 26
33532024-01-10T17:56:25.559ZWrote out file "/tmp/test_repair_verify.data"
33542024-01-10T17:56:25.559Z
33552024-01-10T17:56:25.562ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33562024-01-10T17:56:25.584ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33572024-01-10T17:56:25.584Z 0 000-019 25 25 25 50 50 50 F F F
33582024-01-10T17:56:25.584Z 1 020-039 26 30 30 52 56 56 T F F <---
33592024-01-10T17:56:25.584Z 2 040-059 28 28 28 54 54 54 F T T <---
33602024-01-10T17:56:25.584Z 3 060-079 26 30 30 51 57 57 F F F <---
33612024-01-10T17:56:25.584Z 4 080-099 28 28 28 54 54 54 F T T <---
33622024-01-10T17:56:25.584Z 5 100-119 28 28 28 54 54 54 F F F
33632024-01-10T17:56:25.584Z 6 120-139 28 30 30 54 57 57 F F F <---
33642024-01-10T17:56:25.584Z 7 140-159 14 14 14 21 21 21 F F F
33652024-01-10T17:56:25.584Z 8 160-179 28 28 28 54 54 54 F F F
33662024-01-10T17:56:25.584Z 9 180-199 28 28 28 54 54 54 F F F
33672024-01-10T17:56:25.587Z 10 200-219 28 28 28 54 54 54 F F F
33682024-01-10T17:56:25.587Z 11 220-239 20 20 20 39 39 39 F F F
33692024-01-10T17:56:25.587Z 12 240-259 16 16 16 27 27 27 F F F
33702024-01-10T17:56:25.587Z 13 260-279 19 19 19 33 33 33 F F F
33712024-01-10T17:56:25.588Z 14 280-299 26 26 26 51 51 51 F F F
33722024-01-10T17:56:25.588Z 15 300-319 27 27 27 53 53 53 F T T <---
33732024-01-10T17:56:25.588Z 16 320-339 28 28 28 54 54 54 F F F
33742024-01-10T17:56:25.588Z 17 340-359 12 12 12 19 19 19 F F F
33752024-01-10T17:56:25.588Z 18 360-379 26 26 26 51 51 51 F F F
33762024-01-10T17:56:25.588Z 19 380-399 28 30 30 54 56 56 F F F <---
33772024-01-10T17:56:25.588Z 20 400-419 26 30 30 51 58 58 F F F <---
33782024-01-10T17:56:25.588Z 21 420-439 26 30 30 52 57 57 F F F <---
33792024-01-10T17:56:25.588Z 22 440-459 28 28 28 54 54 54 F F F
33802024-01-10T17:56:25.588Z 23 460-479 28 28 28 54 54 54 F F F
33812024-01-10T17:56:25.588Z 24 480-499 28 30 30 54 58 58 F F F <---
33822024-01-10T17:56:25.588Z 25 500-519 28 30 30 54 58 58 F F F <---
33832024-01-10T17:56:25.588Z 26 520-539 28 28 28 54 54 54 F F F
33842024-01-10T17:56:25.588Z 27 540-559 28 28 28 54 54 54 F F F
33852024-01-10T17:56:25.589Z 28 560-579 28 28 28 54 54 54 F F F
33862024-01-10T17:56:25.589Z 29 580-599 28 30 30 54 57 57 F F F <---
33872024-01-10T17:56:25.589ZMax gen: 30, Max flush: 58
33882024-01-10T17:56:25.589ZError: Difference in extent metadata found!
33892024-01-10T17:56:25.592ZA Difference in extent metadata is expected here
33902024-01-10T17:56:25.592ZOn loop 15
33912024-01-10T17:56:25.592Z
33922024-01-10T17:56:25.592ZVerifying data now
33932024-01-10T17:56:30.780ZNo change after verify
33942024-01-10T17:56:30.780ZLoop: 15 Downstairs dump after verify (and repair):
33952024-01-10T17:56:30.845ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33962024-01-10T17:56:30.845Z 0 000-019 25 25 25 50 50 50 F F F
33972024-01-10T17:56:30.853Z 1 020-039 30 30 30 56 56 56 F F F
33982024-01-10T17:56:30.853Z 2 040-059 31 31 31 59 59 59 F F F
33992024-01-10T17:56:30.853Z 3 060-079 30 30 30 57 57 57 F F F
34002024-01-10T17:56:30.853Z 4 080-099 31 31 31 59 59 59 F F F
34012024-01-10T17:56:30.854Z 5 100-119 28 28 28 54 54 54 F F F
34022024-01-10T17:56:30.854Z 6 120-139 30 30 30 57 57 57 F F F
34032024-01-10T17:56:30.854Z 7 140-159 14 14 14 21 21 21 F F F
34042024-01-10T17:56:30.854Z 8 160-179 28 28 28 54 54 54 F F F
34052024-01-10T17:56:30.854Z 9 180-199 28 28 28 54 54 54 F F F
34062024-01-10T17:56:30.854Z 10 200-219 28 28 28 54 54 54 F F F
34072024-01-10T17:56:30.854Z 11 220-239 20 20 20 39 39 39 F F F
34082024-01-10T17:56:30.855Z 12 240-259 16 16 16 27 27 27 F F F
34092024-01-10T17:56:30.855Z 13 260-279 19 19 19 33 33 33 F F F
34102024-01-10T17:56:30.855Z 14 280-299 26 26 26 51 51 51 F F F
34112024-01-10T17:56:30.855Z 15 300-319 31 31 31 59 59 59 F F F
34122024-01-10T17:56:30.856Z 16 320-339 28 28 28 54 54 54 F F F
34132024-01-10T17:56:30.856Z 17 340-359 12 12 12 19 19 19 F F F
34142024-01-10T17:56:30.856Z 18 360-379 26 26 26 51 51 51 F F F
34152024-01-10T17:56:30.856Z 19 380-399 30 30 30 56 56 56 F F F
34162024-01-10T17:56:30.856Z 20 400-419 30 30 30 58 58 58 F F F
34172024-01-10T17:56:30.856Z 21 420-439 30 30 30 57 57 57 F F F
34182024-01-10T17:56:30.856Z 22 440-459 28 28 28 54 54 54 F F F
34192024-01-10T17:56:30.856Z 23 460-479 28 28 28 54 54 54 F F F
34202024-01-10T17:56:30.856Z 24 480-499 30 30 30 58 58 58 F F F
34212024-01-10T17:56:30.856Z 25 500-519 30 30 30 58 58 58 F F F
34222024-01-10T17:56:30.856Z 26 520-539 28 28 28 54 54 54 F F F
34232024-01-10T17:56:30.856Z 27 540-559 28 28 28 54 54 54 F F F
34242024-01-10T17:56:30.856Z 28 560-579 28 28 28 54 54 54 F F F
34252024-01-10T17:56:30.857Z 29 580-599 30 30 30 57 57 57 F F F
34262024-01-10T17:56:30.857ZMax gen: 31, Max flush: 59
34272024-01-10T17:56:30.857Z
34282024-01-10T17:56:30.857ZBegin loop 16
34292024-01-10T17:56:30.857ZDownstairs to restart: 2
34302024-01-10T17:56:30.857Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.832830352Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283}
34312024-01-10T17:56:30.857ZCrucible runtime is spawned
34322024-01-10T17:56:30.857Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.83365253Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283}
34332024-01-10T17:56:30.858Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.833718073Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283}
34342024-01-10T17:56:30.858Z{"msg":"Crucible a8a2dfa9-df22-401b-b850-3b7d7a63d916 has session id: 058738d7-2965-4e77-a5c7-329a330e89e2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.833749441Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34352024-01-10T17:56:30.858Z{"msg":"Upstairs opts: Upstairs UUID: a8a2dfa9-df22-401b-b850-3b7d7a63d916, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.833808906Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34362024-01-10T17:56:30.858Z{"msg":"Crucible stats registered with UUID: a8a2dfa9-df22-401b-b850-3b7d7a63d916","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.833840302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34372024-01-10T17:56:30.858Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.833969587Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283}
34382024-01-10T17:56:30.858Z{"msg":"a8a2dfa9-df22-401b-b850-3b7d7a63d916 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834011265Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34392024-01-10T17:56:30.858Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834032818Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"0","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34402024-01-10T17:56:30.858Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834060611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"0","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34412024-01-10T17:56:30.858Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834121826Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"1","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34422024-01-10T17:56:30.860Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834203516Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"1","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34432024-01-10T17:56:30.860Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834249327Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"2","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34442024-01-10T17:56:30.860Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834335933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"2","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34452024-01-10T17:56:30.860Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834409353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34462024-01-10T17:56:30.860Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834448392Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34472024-01-10T17:56:30.860Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834472699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34482024-01-10T17:56:30.861Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834497276Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34492024-01-10T17:56:30.861Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.834745333Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34502024-01-10T17:56:30.861Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.83492136Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34512024-01-10T17:56:30.861Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.835463076Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"1","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34522024-01-10T17:56:30.863Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.835599173Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"1","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34532024-01-10T17:56:30.863Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.835660991Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"0","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34542024-01-10T17:56:30.864Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.835690245Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"0","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34552024-01-10T17:56:30.864Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.836294077Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"2","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34562024-01-10T17:56:30.864Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.836353508Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"2","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34572024-01-10T17:56:30.864Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.836380288Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"0","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34582024-01-10T17:56:30.864Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.836442979Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"1","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34592024-01-10T17:56:30.864Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.837351429Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"0","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34602024-01-10T17:56:30.864Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.837416524Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"0","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34612024-01-10T17:56:30.864Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.837495604Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34622024-01-10T17:56:30.864Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.837563566Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"2","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34632024-01-10T17:56:30.864Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.837748188Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"1","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34642024-01-10T17:56:30.864Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.837849117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"1","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34652024-01-10T17:56:30.865Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.837882282Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34662024-01-10T17:56:30.865Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.838473895Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"2","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34672024-01-10T17:56:30.865Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.838555933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"2","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34682024-01-10T17:56:30.865Z{"msg":"[0]R flush_numbers[0..12]: [50, 56, 59, 57, 59, 54, 57, 21, 54, 54, 54, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.838628267Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34692024-01-10T17:56:30.865Z{"msg":"[0]R generation[0..12]: [25, 30, 31, 30, 31, 28, 30, 14, 28, 28, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.838699893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34702024-01-10T17:56:30.865Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.838773776Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34712024-01-10T17:56:30.865ZWait for a query_work_queue command to finish before sending IO
34722024-01-10T17:56:30.867Z{"msg":"[1]R flush_numbers[0..12]: [50, 56, 59, 57, 59, 54, 57, 21, 54, 54, 54, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.83884717Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34732024-01-10T17:56:30.867Z{"msg":"[1]R generation[0..12]: [25, 30, 31, 30, 31, 28, 30, 14, 28, 28, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.838924849Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34742024-01-10T17:56:30.867Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.838973302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34752024-01-10T17:56:30.867Z{"msg":"[2]R flush_numbers[0..12]: [50, 56, 59, 57, 59, 54, 57, 21, 54, 54, 54, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839050221Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34762024-01-10T17:56:30.867Z{"msg":"[2]R generation[0..12]: [25, 30, 31, 30, 31, 28, 30, 14, 28, 28, 28, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839126818Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34772024-01-10T17:56:30.867Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839206083Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34782024-01-10T17:56:30.867Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839280473Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34792024-01-10T17:56:30.867Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839353364Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34802024-01-10T17:56:30.867Z{"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839424396Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34812024-01-10T17:56:30.867ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
34822024-01-10T17:56:30.867Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839497196Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"":"downstairs","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34832024-01-10T17:56:30.868Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839564723Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34842024-01-10T17:56:30.868Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.83959038Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"0","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34852024-01-10T17:56:30.868Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839631322Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"1","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34862024-01-10T17:56:30.868Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839654279Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"client":"2","":"downstairs","session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34872024-01-10T17:56:30.868Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839693672Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34882024-01-10T17:56:30.868Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.83971467Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34892024-01-10T17:56:30.868Z{"msg":"a8a2dfa9-df22-401b-b850-3b7d7a63d916 is now active with session: 058738d7-2965-4e77-a5c7-329a330e89e2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839753838Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34902024-01-10T17:56:30.868Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.83977397Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283,"session_id":"058738d7-2965-4e77-a5c7-329a330e89e2"}
34912024-01-10T17:56:30.868Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:30.839811677Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1283}
34922024-01-10T17:56:30.868ZLoading write count information from file "/tmp/test_repair_verify.data"
34932024-01-10T17:56:30.868ZRun Repair workload
34942024-01-10T17:56:30.869Z01/30 Write block 298 len 36864 data: 5 5 5 5 4 4 4 3 3
34952024-01-10T17:56:30.870Z02/30 Read block 205 len 8192
34962024-01-10T17:56:30.870Z03/30 Write block 526 len 4096 data: 3
34972024-01-10T17:56:30.874Z04/30 Flush
34982024-01-10T17:56:30.874Z05/30 Write block 291 len 8192 data: 6 4
34992024-01-10T17:56:30.874Z06/30 Write block 143 len 36864 data: 4 4 3 3 3 3 3 3 2
35002024-01-10T17:56:30.874Z07/30 Write block 351 len 40960 data: 2 2 2 2 2 2 3 4 3 3
35012024-01-10T17:56:30.874Z08/30 Read block 324 len 4096
35022024-01-10T17:56:30.875Z09/30 Flush
35032024-01-10T17:56:30.875Z10/30 Write block 18 len 40960 data: 7 6 4 5 4 5 5 5 5 5
35042024-01-10T17:56:30.875Z11/30 Write block 59 len 4096 data: 3
35052024-01-10T17:56:30.875Z12/30 Read block 465 len 32768
35062024-01-10T17:56:30.875Z13/30 Write block 79 len 40960 data: 5 4 5 4 5 3 2 2 2 2
35072024-01-10T17:56:30.875Z14/30 Write block 498 len 36864 data: 7 8 5 6 6 7 5 5 5
35082024-01-10T17:56:30.875Z15/30 Read block 175 len 4096
35092024-01-10T17:56:30.875Z16/30 Read block 507 len 8192
35102024-01-10T17:56:30.875Z17/30 Write block 436 len 12288 data: 3 3 3
35112024-01-10T17:56:30.875Z18/30 Write block 252 len 8192 data: 4 3
35122024-01-10T17:56:30.875Z19/30 Read block 498 len 28672
35132024-01-10T17:56:30.875Z20/30 Read block 586 len 8192
35142024-01-10T17:56:30.880Z21/30 Flush
35152024-01-10T17:56:30.883Z22/30 Write block 507 len 12288 data: 6 3 5
35162024-01-10T17:56:30.884Z23/30 Read block 2 len 24576
35172024-01-10T17:56:30.884Z24/30 Flush
35182024-01-10T17:56:30.887Z25/30 Write block 417 len 36864 data: 3 3 5 5 5 5 4 4 3
35192024-01-10T17:56:30.887Z26/30 Read block 459 len 36864
35202024-01-10T17:56:30.893Z27/30 Write block 60 len 28672 data: 3 2 2 2 4 5 6
35212024-01-10T17:56:30.894Z28/30 Read block 19 len 16384
35222024-01-10T17:56:30.898Z29/30 Write block 323 len 40960 data: 4 7 6 4 4 5 4 4 3 3
35232024-01-10T17:56:30.898Z30/30 Write block 209 len 32768 data: 3 3 4 4 4 4 3 4
35242024-01-10T17:56:30.898Z----------------------------------------------------------------
35252024-01-10T17:56:30.899Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:26
35262024-01-10T17:56:30.899ZStates: Active Active Active
35272024-01-10T17:56:30.899ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35282024-01-10T17:56:30.899Z 5 Acked 1004 Write 2 Done Done Done false
35292024-01-10T17:56:30.899Z 6 Acked 1005 Write 9 Done Done Sent false
35302024-01-10T17:56:30.899Z 7 Acked 1006 Write 10 Done Done Sent false
35312024-01-10T17:56:30.899Z 8 Acked 1007 Read 1 Done Done Sent false
35322024-01-10T17:56:30.899Z 9 Acked 1008 Flush 0 Done Done Sent false
35332024-01-10T17:56:30.899Z 10 Acked 1009 Write 10 Done Done Sent false
35342024-01-10T17:56:30.899Z 11 Acked 1010 Write 1 Done Done Sent false
35352024-01-10T17:56:30.899Z 12 Acked 1011 Read 8 Done Done Sent false
35362024-01-10T17:56:30.899Z 13 Acked 1012 Write 10 Done Done Sent false
35372024-01-10T17:56:30.899Z 14 Acked 1013 Write 9 Done Done Sent false
35382024-01-10T17:56:30.900Z 15 Acked 1014 Read 1 Done Done Sent false
35392024-01-10T17:56:30.900Z 16 Acked 1015 Read 2 Done Done Sent false
35402024-01-10T17:56:30.900Z 17 Acked 1016 Write 3 Done Done Sent false
35412024-01-10T17:56:30.900Z 18 Acked 1017 Write 2 Done Done Sent false
35422024-01-10T17:56:30.900Z 19 Acked 1018 Read 7 Done Done Sent false
35432024-01-10T17:56:30.900Z 20 Acked 1019 Read 2 Done Done Sent false
35442024-01-10T17:56:30.901Z 21 Acked 1020 Flush 0 Done Done Sent false
35452024-01-10T17:56:30.901Z 22 Acked 1021 Write 3 Done Done Sent false
35462024-01-10T17:56:30.901Z 23 Acked 1022 Read 6 Done Done Sent false
35472024-01-10T17:56:30.901Z 24 Acked 1023 Flush 0 Done Done Sent false
35482024-01-10T17:56:30.901Z 25 Acked 1024 Write 9 Done Done Sent false
35492024-01-10T17:56:30.901Z 26 Acked 1025 Read 9 Done Done Sent false
35502024-01-10T17:56:30.901Z 27 Acked 1026 Write 7 Done Done Sent false
35512024-01-10T17:56:30.901Z 28 Acked 1027 Read 4 Done Done Sent false
35522024-01-10T17:56:30.901Z 29 Acked 1028 Write 10 Sent Sent Sent false
35532024-01-10T17:56:30.901Z 30 Acked 1029 Write 8 Sent Sent Sent false
35542024-01-10T17:56:30.902Z STATES DS:0 DS:1 DS:2 TOTAL
35552024-01-10T17:56:30.902Z New 0 0 0 0
35562024-01-10T17:56:30.902Z Sent 2 2 25 29
35572024-01-10T17:56:30.902Z Done 24 24 1 49
35582024-01-10T17:56:30.903Z Skipped 0 0 0 0
35592024-01-10T17:56:30.903Z Error 0 0 0 0
35602024-01-10T17:56:30.903ZLast Flush: 1023 1023 1003
35612024-01-10T17:56:30.903ZDownstairs last five completed: 1003 1002 1001 1000
35622024-01-10T17:56:30.903ZUpstairs last five completed: 30 29 28 27 26
35632024-01-10T17:56:30.903ZWrote out file "/tmp/test_repair_verify.data"
35642024-01-10T17:56:30.907Z
35652024-01-10T17:56:30.910ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
35662024-01-10T17:56:30.932ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35672024-01-10T17:56:30.932Z 0 000-019 32 32 25 62 62 50 F F F <---
35682024-01-10T17:56:30.934Z 1 020-039 32 32 30 62 62 56 F F F <---
35692024-01-10T17:56:30.934Z 2 040-059 32 32 31 62 62 59 F F F <---
35702024-01-10T17:56:30.934Z 3 060-079 32 32 30 62 62 57 T T F <---
35712024-01-10T17:56:30.938Z 4 080-099 32 32 31 62 62 59 F F F <---
35722024-01-10T17:56:30.938Z 5 100-119 28 28 28 54 54 54 F F F
35732024-01-10T17:56:30.938Z 6 120-139 30 30 30 57 57 57 F F F
35742024-01-10T17:56:30.938Z 7 140-159 32 32 14 61 61 21 F F F <---
35752024-01-10T17:56:30.938Z 8 160-179 28 28 28 54 54 54 F F F
35762024-01-10T17:56:30.938Z 9 180-199 28 28 28 54 54 54 F F F
35772024-01-10T17:56:30.938Z 10 200-219 28 28 28 54 54 54 T T F <---
35782024-01-10T17:56:30.938Z 11 220-239 20 20 20 39 39 39 F F F
35792024-01-10T17:56:30.938Z 12 240-259 32 32 16 62 62 27 F F F <---
35802024-01-10T17:56:30.938Z 13 260-279 19 19 19 33 33 33 F F F
35812024-01-10T17:56:30.938Z 14 280-299 32 32 32 61 61 60 F F T <---
35822024-01-10T17:56:30.938Z 15 300-319 32 32 32 60 60 60 F F F
35832024-01-10T17:56:30.939Z 16 320-339 28 28 28 54 54 54 T T F <---
35842024-01-10T17:56:30.939Z 17 340-359 32 32 12 61 61 19 F F F <---
35852024-01-10T17:56:30.939Z 18 360-379 32 32 26 61 61 51 F F F <---
35862024-01-10T17:56:30.939Z 19 380-399 30 30 30 56 56 56 F F F
35872024-01-10T17:56:30.939Z 20 400-419 30 30 30 58 58 58 T T F <---
35882024-01-10T17:56:30.939Z 21 420-439 32 32 30 62 62 57 T T F <---
35892024-01-10T17:56:30.939Z 22 440-459 28 28 28 54 54 54 F F F
35902024-01-10T17:56:30.939Z 23 460-479 28 28 28 54 54 54 F F F
35912024-01-10T17:56:30.939Z 24 480-499 32 32 30 62 62 58 F F F <---
35922024-01-10T17:56:30.939Z 25 500-519 32 32 30 63 63 58 F F F <---
35932024-01-10T17:56:30.939Z 26 520-539 32 32 32 60 60 60 F F F
35942024-01-10T17:56:30.939Z 27 540-559 28 28 28 54 54 54 F F F
35952024-01-10T17:56:30.940Z 28 560-579 28 28 28 54 54 54 F F F
35962024-01-10T17:56:30.940Z 29 580-599 30 30 30 57 57 57 F F F
35972024-01-10T17:56:30.940ZMax gen: 32, Max flush: 63
35982024-01-10T17:56:30.940ZError: Difference in extent metadata found!
35992024-01-10T17:56:30.940ZA Difference in extent metadata is expected here
36002024-01-10T17:56:30.940ZOn loop 16
36012024-01-10T17:56:30.940Z
36022024-01-10T17:56:30.944ZVerifying data now
36032024-01-10T17:56:36.457ZNo change after verify
36042024-01-10T17:56:36.457ZLoop: 16 Downstairs dump after verify (and repair):
36052024-01-10T17:56:36.476ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36062024-01-10T17:56:36.476Z 0 000-019 32 32 32 62 62 62 F F F
36072024-01-10T17:56:36.476Z 1 020-039 32 32 32 62 62 62 F F F
36082024-01-10T17:56:36.476Z 2 040-059 32 32 32 62 62 62 F F F
36092024-01-10T17:56:36.476Z 3 060-079 33 33 33 64 64 64 F F F
36102024-01-10T17:56:36.476Z 4 080-099 32 32 32 62 62 62 F F F
36112024-01-10T17:56:36.479Z 5 100-119 28 28 28 54 54 54 F F F
36122024-01-10T17:56:36.479Z 6 120-139 30 30 30 57 57 57 F F F
36132024-01-10T17:56:36.480Z 7 140-159 32 32 32 61 61 61 F F F
36142024-01-10T17:56:36.480Z 8 160-179 28 28 28 54 54 54 F F F
36152024-01-10T17:56:36.480Z 9 180-199 28 28 28 54 54 54 F F F
36162024-01-10T17:56:36.480Z 10 200-219 33 33 33 64 64 64 F F F
36172024-01-10T17:56:36.480Z 11 220-239 20 20 20 39 39 39 F F F
36182024-01-10T17:56:36.480Z 12 240-259 32 32 32 62 62 62 F F F
36192024-01-10T17:56:36.480Z 13 260-279 19 19 19 33 33 33 F F F
36202024-01-10T17:56:36.480Z 14 280-299 32 32 32 61 61 61 F F F
36212024-01-10T17:56:36.480Z 15 300-319 32 32 32 60 60 60 F F F
36222024-01-10T17:56:36.480Z 16 320-339 33 33 33 64 64 64 F F F
36232024-01-10T17:56:36.482Z 17 340-359 32 32 32 61 61 61 F F F
36242024-01-10T17:56:36.482Z 18 360-379 32 32 32 61 61 61 F F F
36252024-01-10T17:56:36.482Z 19 380-399 30 30 30 56 56 56 F F F
36262024-01-10T17:56:36.482Z 20 400-419 33 33 33 64 64 64 F F F
36272024-01-10T17:56:36.482Z 21 420-439 33 33 33 64 64 64 F F F
36282024-01-10T17:56:36.482Z 22 440-459 28 28 28 54 54 54 F F F
36292024-01-10T17:56:36.482Z 23 460-479 28 28 28 54 54 54 F F F
36302024-01-10T17:56:36.482Z 24 480-499 32 32 32 62 62 62 F F F
36312024-01-10T17:56:36.482Z 25 500-519 32 32 32 63 63 63 F F F
36322024-01-10T17:56:36.482Z 26 520-539 32 32 32 60 60 60 F F F
36332024-01-10T17:56:36.482Z 27 540-559 28 28 28 54 54 54 F F F
36342024-01-10T17:56:36.482Z 28 560-579 28 28 28 54 54 54 F F F
36352024-01-10T17:56:36.483Z 29 580-599 30 30 30 57 57 57 F F F
36362024-01-10T17:56:36.483ZMax gen: 33, Max flush: 64
36372024-01-10T17:56:36.486Z
36382024-01-10T17:56:36.486ZBegin loop 17
36392024-01-10T17:56:36.486ZDownstairs to restart: 0
36402024-01-10T17:56:36.510Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.508563705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291}
36412024-01-10T17:56:36.510Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.509102949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291}
36422024-01-10T17:56:36.510Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.509152307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291}
36432024-01-10T17:56:36.510Z{"msg":"Crucible 50c3dc8f-c7f8-4608-b0e7-50868a619b1b has session id: 2837a322-d588-4b89-8136-11a84d22ea09","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.509177199Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36442024-01-10T17:56:36.515Z{"msg":"Upstairs opts: Upstairs UUID: 50c3dc8f-c7f8-4608-b0e7-50868a619b1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.50922887Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36452024-01-10T17:56:36.515Z{"msg":"Crucible stats registered with UUID: 50c3dc8f-c7f8-4608-b0e7-50868a619b1b","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.509355568Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36462024-01-10T17:56:36.515ZCrucible runtime is spawned
36472024-01-10T17:56:36.515Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.509610801Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291}
36482024-01-10T17:56:36.515Z{"msg":"50c3dc8f-c7f8-4608-b0e7-50868a619b1b active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.509818479Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36492024-01-10T17:56:36.515Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.509892611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"0","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36502024-01-10T17:56:36.515Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.509920031Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"0","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36512024-01-10T17:56:36.516Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.509987064Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"1","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36522024-01-10T17:56:36.516Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.51006211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"1","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36532024-01-10T17:56:36.516Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.510107234Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"2","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36542024-01-10T17:56:36.516Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.510293897Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"2","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36552024-01-10T17:56:36.516Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.510412356Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36562024-01-10T17:56:36.516Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.510518641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36572024-01-10T17:56:36.516Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.510551943Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36582024-01-10T17:56:36.516Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.510593627Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36592024-01-10T17:56:36.516Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.510705908Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36602024-01-10T17:56:36.516Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.510922366Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36612024-01-10T17:56:36.516Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.512053299Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"1","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36622024-01-10T17:56:36.516Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.512119743Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"1","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36632024-01-10T17:56:36.516Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.51214666Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"2","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36642024-01-10T17:56:36.517Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.512203965Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"2","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36652024-01-10T17:56:36.517Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.512280254Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"0","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36662024-01-10T17:56:36.517Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.512386958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"0","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36672024-01-10T17:56:36.517Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.513022597Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"1","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36682024-01-10T17:56:36.517Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.513169284Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"2","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36692024-01-10T17:56:36.522Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.513561495Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"0","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36702024-01-10T17:56:36.522Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514016112Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"2","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36712024-01-10T17:56:36.522Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514108109Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"2","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36722024-01-10T17:56:36.522Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514151994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36732024-01-10T17:56:36.523Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514211234Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"1","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36742024-01-10T17:56:36.523Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514252124Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"1","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36752024-01-10T17:56:36.523Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514330422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36762024-01-10T17:56:36.523Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514504302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"0","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36772024-01-10T17:56:36.523Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514569251Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"0","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36782024-01-10T17:56:36.523Z{"msg":"[0]R flush_numbers[0..12]: [62, 62, 62, 64, 62, 54, 57, 61, 54, 54, 64, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514596892Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36792024-01-10T17:56:36.523Z{"msg":"[0]R generation[0..12]: [32, 32, 32, 33, 32, 28, 30, 32, 28, 28, 33, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514640481Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36802024-01-10T17:56:36.523Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514666645Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36812024-01-10T17:56:36.523Z{"msg":"[1]R flush_numbers[0..12]: [62, 62, 62, 64, 62, 54, 57, 61, 54, 54, 64, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514708267Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36822024-01-10T17:56:36.523Z{"msg":"[1]R generation[0..12]: [32, 32, 32, 33, 32, 28, 30, 32, 28, 28, 33, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514732878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36832024-01-10T17:56:36.523Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514779302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36842024-01-10T17:56:36.524Z{"msg":"[2]R flush_numbers[0..12]: [62, 62, 62, 64, 62, 54, 57, 61, 54, 54, 64, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514803997Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
36852024-01-10T17:56:36.524Z,"":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36862024-01-10T17:56:36.524Z{"msg":"[2]R generation[0..12]: [32, 32, 32, 33, 32, 28, 30, 32, 28, 28, 33, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514913551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36872024-01-10T17:56:36.524Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.514944556Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36882024-01-10T17:56:36.524Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515005401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36892024-01-10T17:56:36.524Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515039206Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36902024-01-10T17:56:36.524Z{"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515063145Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36912024-01-10T17:56:36.524Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515084884Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"":"downstairs","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36922024-01-10T17:56:36.524Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515106507Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36932024-01-10T17:56:36.524Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515145759Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"0","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36942024-01-10T17:56:36.524Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515169643Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"1","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36952024-01-10T17:56:36.524Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.5151913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"client":"2","":"downstairs","session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36962024-01-10T17:56:36.524Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.51523102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36972024-01-10T17:56:36.524Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515253454Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36982024-01-10T17:56:36.524Z{"msg":"50c3dc8f-c7f8-4608-b0e7-50868a619b1b is now active with session: 2837a322-d588-4b89-8136-11a84d22ea09","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515300173Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
36992024-01-10T17:56:36.524Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515352524Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291,"session_id":"2837a322-d588-4b89-8136-11a84d22ea09"}
37002024-01-10T17:56:36.524Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:36.515375545Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1291}
37012024-01-10T17:56:36.524ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37022024-01-10T17:56:36.524ZLoading write count information from file "/tmp/test_repair_verify.data"
37032024-01-10T17:56:36.524ZRun Repair workload
37042024-01-10T17:56:36.524Z01/30 Write block 205 len 16384 data: 2 2 2 3
37052024-01-10T17:56:36.524Z02/30 Read block 271 len 40960
37062024-01-10T17:56:36.540Z03/30 Read block 43 len 28672
37072024-01-10T17:56:36.540Z04/30 Read block 337 len 32768
37082024-01-10T17:56:36.540Z05/30 Write block 359 len 4096 data: 4
37092024-01-10T17:56:36.540Z06/30 Write block 278 len 32768 data: 3 3 3 3 3 3 3 3
37102024-01-10T17:56:36.540Z07/30 Write block 291 len 20480 data: 7 5 4 3 3
37112024-01-10T17:56:36.540Z08/30 Read block 292 len 20480
37122024-01-10T17:56:36.540Z09/30 Read block 582 len 4096
37132024-01-10T17:56:36.540Z10/30 Read block 343 len 40960
37142024-01-10T17:56:36.540Z11/30 Write block 445 len 24576 data: 9 9 6 5 4 4
37152024-01-10T17:56:36.541Z12/30 Read block 65 len 4096
37162024-01-10T17:56:36.545Z13/30 Flush
37172024-01-10T17:56:36.545Z14/30 Write block 95 len 8192 data: 5 5
37182024-01-10T17:56:36.548Z15/30 Write block 531 len 24576 data: 4 4 4 3 3 4
37192024-01-10T17:56:36.548Z16/30 Write block 401 len 32768 data: 4 2 2 2 2 3 4 4
37202024-01-10T17:56:36.548Z17/30 Read block 54 len 36864
37212024-01-10T17:56:36.551Z18/30 Read block 412 len 24576
37222024-01-10T17:56:36.554Z19/30 Read block 482 len 20480
37232024-01-10T17:56:36.557Z20/30 Read block 340 len 4096
37242024-01-10T17:56:36.557Z21/30 Write block 576 len 4096 data: 5
37252024-01-10T17:56:36.560Z22/30 Write block 410 len 8192 data: 5 6
37262024-01-10T17:56:36.560Z23/30 Write block 419 len 36864 data: 6 6 6 6 5 5 4 3 3
37272024-01-10T17:56:36.560Z24/30 Write block 414 len 32768 data: 6 4 3 4 4 7 7 7
37282024-01-10T17:56:36.563Z25/30 Read block 80 len 20480
37292024-01-10T17:56:36.563Z26/30 Read block 566 len 40960
37302024-01-10T17:56:36.566Z27/30 Read block 235 len 8192
37312024-01-10T17:56:36.569Z28/30 Read block 31 len 32768
37322024-01-10T17:56:36.573Z29/30 Write block 35 len 16384 data: 3 3 2 2
37332024-01-10T17:56:36.573Z30/30 Write block 297 len 12288 data: 4 6 6
37342024-01-10T17:56:36.574Z----------------------------------------------------------------
37352024-01-10T17:56:36.575Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37362024-01-10T17:56:36.575ZStates: Active Active Active
37372024-01-10T17:56:36.575ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37382024-01-10T17:56:36.575Z 1 Acked 1000 Write 4 Done Done Done false
37392024-01-10T17:56:36.575Z 2 Acked 1001 Read 10 Sent Done Done false
37402024-01-10T17:56:36.575Z 3 Acked 1002 Read 7 Sent Done Done false
37412024-01-10T17:56:36.575Z 4 Acked 1003 Read 8 Sent Done Done false
37422024-01-10T17:56:36.575Z 5 Acked 1004 Write 1 Sent Done Done false
37432024-01-10T17:56:36.575Z 6 Acked 1005 Write 8 Sent Done Done false
37442024-01-10T17:56:36.576Z 7 Acked 1006 Write 5 Sent Done Done false
37452024-01-10T17:56:36.576Z 8 Acked 1007 Read 5 Sent Done Done false
37462024-01-10T17:56:36.576Z 9 Acked 1008 Read 1 Sent Done Done false
37472024-01-10T17:56:36.576Z 10 Acked 1009 Read 10 Sent Done Done false
37482024-01-10T17:56:36.576Z 11 Acked 1010 Write 6 Sent Done Done false
37492024-01-10T17:56:36.576Z 12 Acked 1011 Read 1 Sent Done Done false
37502024-01-10T17:56:36.576Z 13 Acked 1012 Flush 0 Sent Done Done false
37512024-01-10T17:56:36.576Z 14 Acked 1013 Write 2 Sent Done Done false
37522024-01-10T17:56:36.576Z 15 Acked 1014 Write 6 Sent Done Done false
37532024-01-10T17:56:36.576Z 16 Acked 1015 Write 8 Sent Done Done false
37542024-01-10T17:56:36.576Z 17 Acked 1016 Read 9 Sent Done Done false
37552024-01-10T17:56:36.576Z 18 Acked 1017 Read 6 Sent Done Done false
37562024-01-10T17:56:36.576Z 19 Acked 1018 Read 5 Sent Done Done false
37572024-01-10T17:56:36.576Z 20 Acked 1019 Read 1 Sent Done Done false
37582024-01-10T17:56:36.576Z 21 Acked 1020 Write 1 Sent Done Done false
37592024-01-10T17:56:36.576Z 22 Acked 1021 Write 2 Sent Done Done false
37602024-01-10T17:56:36.576Z 23 Acked 1022 Write 9 Sent Done Done false
37612024-01-10T17:56:36.576Z 24 Acked 1023 Write 8 Sent Done Done false
37622024-01-10T17:56:36.576Z 25 Acked 1024 Read 5 Sent Done Done false
37632024-01-10T17:56:36.576Z 26 Acked 1025 Read 10 Sent Done Done false
37642024-01-10T17:56:36.576Z 27 Acked 1026 Read 2 Sent Done Done false
37652024-01-10T17:56:36.576Z 28 Acked 1027 Read 8 Sent Done Done false
37662024-01-10T17:56:36.576Z 29 Acked 1028 Write 4 Sent Done Done false
37672024-01-10T17:56:36.577Z 30 Acked 1029 Write 3 Sent Sent Sent false
37682024-01-10T17:56:36.577Z STATES DS:0 DS:1 DS:2 TOTAL
37692024-01-10T17:56:36.577Z New 0 0 0 0
37702024-01-10T17:56:36.577Z Sent 29 1 1 31
37712024-01-10T17:56:36.577Z Done 1 29 29 59
37722024-01-10T17:56:36.577Z Skipped 0 0 0 0
37732024-01-10T17:56:36.577Z Error 0 0 0 0
37742024-01-10T17:56:36.577ZLast Flush: 0 1012 1012
37752024-01-10T17:56:36.577ZDownstairs last five completed:
37762024-01-10T17:56:36.580ZUpstairs last five completed: 30 29 28 27 26
37772024-01-10T17:56:36.580ZWrote out file "/tmp/test_repair_verify.data"
37782024-01-10T17:56:36.580Z
37792024-01-10T17:56:36.587ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
37802024-01-10T17:56:36.613ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
37812024-01-10T17:56:36.613Z 0 000-019 32 32 32 62 62 62 F F F
37822024-01-10T17:56:36.614Z 1 020-039 32 32 32 62 62 62 F T T <---
37832024-01-10T17:56:36.615Z 2 040-059 32 32 32 62 62 62 F F F
37842024-01-10T17:56:36.615Z 3 060-079 33 33 33 64 64 64 F F F
37852024-01-10T17:56:36.615Z 4 080-099 32 32 32 62 62 62 F T T <---
37862024-01-10T17:56:36.615Z 5 100-119 28 28 28 54 54 54 F F F
37872024-01-10T17:56:36.621Z 6 120-139 30 30 30 57 57 57 F F F
37882024-01-10T17:56:36.621Z 7 140-159 32 32 32 61 61 61 F F F
37892024-01-10T17:56:36.622Z 8 160-179 28 28 28 54 54 54 F F F
37902024-01-10T17:56:36.622Z 9 180-199 28 28 28 54 54 54 F F F
37912024-01-10T17:56:36.622Z 10 200-219 33 34 34 64 65 65 T F F <---
37922024-01-10T17:56:36.622Z 11 220-239 20 20 20 39 39 39 F F F
37932024-01-10T17:56:36.622Z 12 240-259 32 32 32 62 62 62 F F F
37942024-01-10T17:56:36.622Z 13 260-279 19 34 34 33 65 65 F F F <---
37952024-01-10T17:56:36.622Z 14 280-299 32 34 34 61 65 65 F T T <---
37962024-01-10T17:56:36.622Z 15 300-319 32 32 32 60 60 60 F F F
37972024-01-10T17:56:36.623Z 16 320-339 33 33 33 64 64 64 F F F
37982024-01-10T17:56:36.623Z 17 340-359 32 34 34 61 65 65 F F F <---
37992024-01-10T17:56:36.623Z 18 360-379 32 32 32 61 61 61 F F F
38002024-01-10T17:56:36.623Z 19 380-399 30 30 30 56 56 56 F F F
38012024-01-10T17:56:36.623Z 20 400-419 33 33 33 64 64 64 F T T <---
38022024-01-10T17:56:36.623Z 21 420-439 33 33 33 64 64 64 F T T <---
38032024-01-10T17:56:36.623Z 22 440-459 28 34 34 54 65 65 F F F <---
38042024-01-10T17:56:36.623Z 23 460-479 28 28 28 54 54 54 F F F
38052024-01-10T17:56:36.623Z 24 480-499 32 32 32 62 62 62 F F F
38062024-01-10T17:56:36.623Z 25 500-519 32 32 32 63 63 63 F F F
38072024-01-10T17:56:36.626Z 26 520-539 32 32 32 60 60 60 F T T <---
38082024-01-10T17:56:36.626Z 27 540-559 28 28 28 54 54 54 F F F
38092024-01-10T17:56:36.626Z 28 560-579 28 28 28 54 54 54 F T T <---
38102024-01-10T17:56:36.626Z 29 580-599 30 30 30 57 57 57 F F F
38112024-01-10T17:56:36.626ZMax gen: 34, Max flush: 65
38122024-01-10T17:56:36.626ZError: Difference in extent metadata found!
38132024-01-10T17:56:36.627ZA Difference in extent metadata is expected here
38142024-01-10T17:56:36.627ZOn loop 17
38152024-01-10T17:56:36.627Z
38162024-01-10T17:56:36.630ZVerifying data now
38172024-01-10T17:56:41.752ZNo change after verify
38182024-01-10T17:56:41.753ZLoop: 17 Downstairs dump after verify (and repair):
38192024-01-10T17:56:41.774ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38202024-01-10T17:56:41.774Z 0 000-019 32 32 32 62 62 62 F F F
38212024-01-10T17:56:41.775Z 1 020-039 35 35 35 66 66 66 F F F
38222024-01-10T17:56:41.775Z 2 040-059 32 32 32 62 62 62 F F F
38232024-01-10T17:56:41.775Z 3 060-079 33 33 33 64 64 64 F F F
38242024-01-10T17:56:41.778Z 4 080-099 35 35 35 66 66 66 F F F
38252024-01-10T17:56:41.778Z 5 100-119 28 28 28 54 54 54 F F F
38262024-01-10T17:56:41.778Z 6 120-139 30 30 30 57 57 57 F F F
38272024-01-10T17:56:41.778Z 7 140-159 32 32 32 61 61 61 F F F
38282024-01-10T17:56:41.778Z 8 160-179 28 28 28 54 54 54 F F F
38292024-01-10T17:56:41.779Z 9 180-199 28 28 28 54 54 54 F F F
38302024-01-10T17:56:41.779Z 10 200-219 34 34 34 65 65 65 F F F
38312024-01-10T17:56:41.779Z 11 220-239 20 20 20 39 39 39 F F F
38322024-01-10T17:56:41.779Z 12 240-259 32 32 32 62 62 62 F F F
38332024-01-10T17:56:41.779Z 13 260-279 34 34 34 65 65 65 F F F
38342024-01-10T17:56:41.779Z 14 280-299 35 35 35 66 66 66 F F F
38352024-01-10T17:56:41.779Z 15 300-319 32 32 32 60 60 60 F F F
38362024-01-10T17:56:41.779Z 16 320-339 33 33 33 64 64 64 F F F
38372024-01-10T17:56:41.779Z 17 340-359 34 34 34 65 65 65 F F F
38382024-01-10T17:56:41.779Z 18 360-379 32 32 32 61 61 61 F F F
38392024-01-10T17:56:41.779Z 19 380-399 30 30 30 56 56 56 F F F
38402024-01-10T17:56:41.779Z 20 400-419 35 35 35 66 66 66 F F F
38412024-01-10T17:56:41.780Z 21 420-439 35 35 35 66 66 66 F F F
38422024-01-10T17:56:41.780Z 22 440-459 34 34 34 65 65 65 F F F
38432024-01-10T17:56:41.780Z 23 460-479 28 28 28 54 54 54 F F F
38442024-01-10T17:56:41.780Z 24 480-499 32 32 32 62 62 62 F F F
38452024-01-10T17:56:41.780Z 25 500-519 32 32 32 63 63 63 F F F
38462024-01-10T17:56:41.780Z 26 520-539 35 35 35 66 66 66 F F F
38472024-01-10T17:56:41.780Z 27 540-559 28 28 28 54 54 54 F F F
38482024-01-10T17:56:41.780Z 28 560-579 35 35 35 66 66 66 F F F
38492024-01-10T17:56:41.780Z 29 580-599 30 30 30 57 57 57 F F F
38502024-01-10T17:56:41.780ZMax gen: 35, Max flush: 66
38512024-01-10T17:56:41.784Z
38522024-01-10T17:56:41.784ZBegin loop 18
38532024-01-10T17:56:41.785ZDownstairs to restart: 2
38542024-01-10T17:56:41.811Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.808857871Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299}
38552024-01-10T17:56:41.811Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.809426712Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299}
38562024-01-10T17:56:41.811Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.809498527Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299}
38572024-01-10T17:56:41.818Z{"msg":"Crucible 445f467e-b77a-4120-839f-2557133931b8 has session id: 25ea3127-7673-4fd4-a5ad-703c53b56c64","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.809571398Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38582024-01-10T17:56:41.819Z{Crucible runtime is spawned
38592024-01-10T17:56:41.820Z"msg":"Upstairs opts: Upstairs UUID: 445f467e-b77a-4120-839f-2557133931b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.80984379Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38602024-01-10T17:56:41.820Z{"msg":"Crucible stats registered with UUID: 445f467e-b77a-4120-839f-2557133931b8","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.809996115Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38612024-01-10T17:56:41.820Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810056376Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299}
38622024-01-10T17:56:41.820Z{"msg":"445f467e-b77a-4120-839f-2557133931b8 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810090788Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38632024-01-10T17:56:41.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810131052Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"0","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38642024-01-10T17:56:41.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810163579Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"0","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38652024-01-10T17:56:41.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810204371Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"1","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38662024-01-10T17:56:41.821Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810245737Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"1","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38672024-01-10T17:56:41.821Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810344076Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"2","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38682024-01-10T17:56:41.821Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810498037Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"2","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38692024-01-10T17:56:41.821Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810752028Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38702024-01-10T17:56:41.821Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810799123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38712024-01-10T17:56:41.821Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810867438Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38722024-01-10T17:56:41.821Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810912079Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38732024-01-10T17:56:41.821Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.810983053Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38742024-01-10T17:56:41.821Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.811024913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38752024-01-10T17:56:41.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.81180259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"0","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38762024-01-10T17:56:41.821Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.811862705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"0","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38772024-01-10T17:56:41.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.812010984Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"1","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38782024-01-10T17:56:41.822Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.812134183Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"1","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38792024-01-10T17:56:41.823Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.812201107Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"2","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38802024-01-10T17:56:41.823Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.812227456Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"2","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38812024-01-10T17:56:41.823Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.812948144Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"0","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38822024-01-10T17:56:41.823Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.813061056Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"1","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38832024-01-10T17:56:41.823Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.81312181Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"2","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38842024-01-10T17:56:41.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.81384068Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"0","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38852024-01-10T17:56:41.824Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.813905732Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"0","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38862024-01-10T17:56:41.824Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.813987976Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38872024-01-10T17:56:41.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814160431Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"1","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38882024-01-10T17:56:41.824Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814217903Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"1","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38892024-01-10T17:56:41.824Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814243232Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38902024-01-10T17:56:41.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814282307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"2","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38912024-01-10T17:56:41.824Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814306381Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"2","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38922024-01-10T17:56:41.824Z{"msg":"[0]R flush_numbers[0..12]: [62, 66, 62, 64, 66, 54, 57, 61, 54, 54, 65, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.81432973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38932024-01-10T17:56:41.824Z{"msg":"[0]R generation[0..12]: [32, 35, 32, 33, 35, 28, 30, 32, 28, 28, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814353367Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38942024-01-10T17:56:41.824Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814429431Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38952024-01-10T17:56:41.824Z{"msg":"[1]R flush_numbers[0..12]: [62, 66, 62, 64, 66, 54, 57, 61, 54, 54, 65, 39]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
38962024-01-10T17:56:41.824Z,"time":"2024-01-10T17:56:41.814454897Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38972024-01-10T17:56:41.825Z{"msg":"[1]R generation[0..12]: [32, 35, 32, 33, 35, 28, 30, 32, 28, 28, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814477994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38982024-01-10T17:56:41.825Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814501516Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
38992024-01-10T17:56:41.825Z{"msg":"[2]R flush_numbers[0..12]: [62, 66, 62, 64, 66, 54, 57, 61, 54, 54, 65, 39]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814543669Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39002024-01-10T17:56:41.826Z{"msg":"[2]R generation[0..12]: [32, 35, 32, 33, 35, 28, 30, 32, 28, 28, 34, 20]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814568097Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39012024-01-10T17:56:41.826Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814615529Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39022024-01-10T17:56:41.826Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814638733Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39032024-01-10T17:56:41.826Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814660633Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39042024-01-10T17:56:41.826Z{"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814681352Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39052024-01-10T17:56:41.830Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814743655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"":"downstairs","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39062024-01-10T17:56:41.830Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814822704Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39072024-01-10T17:56:41.831Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.814896154Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"0","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39082024-01-10T17:56:41.831Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.8149378Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"1","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39092024-01-10T17:56:41.831Z{"msg":"Transition from WaitQuorum to Active","v":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39102024-01-10T17:56:41.831Z0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.815109671Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"client":"2","":"downstairs","session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39112024-01-10T17:56:41.831Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.81516906Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39122024-01-10T17:56:41.831Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.815192694Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39132024-01-10T17:56:41.831Z{"msg":"445f467e-b77a-4120-839f-2557133931b8 is now active with session: 25ea3127-7673-4fd4-a5ad-703c53b56c64","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.815214897Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39142024-01-10T17:56:41.831Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.815234771Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299,"session_id":"25ea3127-7673-4fd4-a5ad-703c53b56c64"}
39152024-01-10T17:56:41.831Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:41.815273665Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1299}
39162024-01-10T17:56:41.831ZLoading write count information from file "/tmp/test_repair_verify.data"
39172024-01-10T17:56:41.831ZRun Repair workload
39182024-01-10T17:56:41.831Z01/30 Write block 505 len 36864 data: 6 6 7 4 6 5 5 7 8
39192024-01-10T17:56:41.832Z02/30 Read block 337 len 36864
39202024-01-10T17:56:41.832Z03/30 Flush
39212024-01-10T17:56:41.832Z04/30 Write block 291 len 40960 data: 8 6 5 4 4 4 5 7 7 6
39222024-01-10T17:56:41.832Z05/30 Write block 466 len 8192 data: 5 6
39232024-01-10T17:56:41.832Z06/30 Read block 394 len 20480
39242024-01-10T17:56:41.836Z07/30 Read block 447 len 28672
39252024-01-10T17:56:41.837Z08/30 Read block 298 len 16384
39262024-01-10T17:56:41.837Z09/30 Flush
39272024-01-10T17:56:41.841Z10/30 Write block 290 len 4096 data: 5
39282024-01-10T17:56:41.841Z11/30 Flush
39292024-01-10T17:56:41.841Z12/30 Write block 311 len 4096 data: 2
39302024-01-10T17:56:41.842Z13/30 Read block 124 len 32768
39312024-01-10T17:56:41.846Z14/30 Write block 548 len 8192 data: 5 5
39322024-01-10T17:56:41.846Z15/30 Write block 484 len 4096 data: 4
39332024-01-10T17:56:41.846Z16/30 Write block 502 len 24576 data: 7 8 6 7 7 8
39342024-01-10T17:56:41.846Z17/30 Read block 451 len 8192
39352024-01-10T17:56:41.850Z18/30 Read block 339 len 24576
39362024-01-10T17:56:41.853Z19/30 Read block 130 len 20480
39372024-01-10T17:56:41.853Z20/30 Write block 552 len 28672 data: 5 4 4 3 3 4 3
39382024-01-10T17:56:41.857Z21/30 Write block 574 len 20480 data: 4 4 6 5 4
39392024-01-10T17:56:41.857Z22/30 Write block 402 len 32768 data: 3 3 3 3 4 5 5 4
39402024-01-10T17:56:41.858Z23/30 Write block 266 len 24576 data: 3 4 4 4 5 5
39412024-01-10T17:56:41.863Z24/30 Write block 315 len 28672 data: 3 4 3 3 4 4 4
39422024-01-10T17:56:41.863Z25/30 Write block 180 len 28672 data: 5 5 5 5 5 5 4
39432024-01-10T17:56:41.863Z26/30 Write block 225 len 24576 data: 2 2 2 2 2 2
39442024-01-10T17:56:41.863Z27/30 Write block 370 len 12288 data: 3 4 3
39452024-01-10T17:56:41.863Z28/30 Write block 540 len 20480 data: 4 4 3 3 4
39462024-01-10T17:56:41.867Z29/30 Write block 160 len 40960 data: 3 4 7 7 7 7 6 5 4 3
39472024-01-10T17:56:41.868Z30/30 Write block 155 len 12288 data: 2 2 2
39482024-01-10T17:56:41.868Z----------------------------------------------------------------
39492024-01-10T17:56:41.868Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
39502024-01-10T17:56:41.868ZStates: Active Active Active
39512024-01-10T17:56:41.868ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
39522024-01-10T17:56:41.868Z 1 Acked 1000 Write 9 Done Done Sent false
39532024-01-10T17:56:41.868Z 2 Acked 1001 Read 9 Done Done Sent false
39542024-01-10T17:56:41.868Z 3 Acked 1002 Flush 0 Done Done Sent false
39552024-01-10T17:56:41.868Z 4 Acked 1003 Write 10 Done Done Sent false
39562024-01-10T17:56:41.868Z 5 Acked 1004 Write 2 Done Done Sent false
39572024-01-10T17:56:41.868Z 6 Acked 1005 Read 5 Done Done Sent false
39582024-01-10T17:56:41.869Z 7 Acked 1006 Read 7 Done Done Sent false
39592024-01-10T17:56:41.869Z 8 Acked 1007 Read 4 Done Done Sent false
39602024-01-10T17:56:41.869Z 9 Acked 1008 Flush 0 Done Done Sent false
39612024-01-10T17:56:41.869Z 10 Acked 1009 Write 1 Done Done Sent false
39622024-01-10T17:56:41.869Z 11 Acked 1010 Flush 0 Done Done Sent false
39632024-01-10T17:56:41.869Z 12 Acked 1011 Write 1 Done Done Sent false
39642024-01-10T17:56:41.869Z 13 Acked 1012 Read 8 Done Done Sent false
39652024-01-10T17:56:41.869Z 14 Acked 1013 Write 2 Done Done Sent false
39662024-01-10T17:56:41.869Z 15 Acked 1014 Write 1 Done Done Sent false
39672024-01-10T17:56:41.869Z 16 Acked 1015 Write 6 Done Done Sent false
39682024-01-10T17:56:41.869Z 17 Acked 1016 Read 2 Done Done Sent false
39692024-01-10T17:56:41.869Z 18 Acked 1017 Read 6 Done Done Sent false
39702024-01-10T17:56:41.870Z 19 Acked 1018 Read 5 Done Done Sent false
39712024-01-10T17:56:41.870Z 20 Acked 1019 Write 7 Done Done Sent false
39722024-01-10T17:56:41.870Z 21 Acked 1020 Write 5 Done Done Sent false
39732024-01-10T17:56:41.870Z 22 Acked 1021 Write 8 Done Done Sent false
39742024-01-10T17:56:41.870Z 23 Acked 1022 Write 6 Done Done Sent false
39752024-01-10T17:56:41.870Z 24 Acked 1023 Write 7 Done Done Sent false
39762024-01-10T17:56:41.870Z 25 Acked 1024 Write 7 Done Done Sent false
39772024-01-10T17:56:41.870Z 26 Acked 1025 Write 6 Done Done Sent false
39782024-01-10T17:56:41.870Z 27 Acked 1026 Write 3 Done Done Sent false
39792024-01-10T17:56:41.870Z 28 Acked 1027 Write 5 Done Done Sent false
39802024-01-10T17:56:41.870Z 29 Acked 1028 Write 10 Sent Sent Sent false
39812024-01-10T17:56:41.870Z 30 Acked 1029 Write 3 Sent Sent Sent false
39822024-01-10T17:56:41.870Z STATES DS:0 DS:1 DS:2 TOTAL
39832024-01-10T17:56:41.871Z New 0 0 0 0
39842024-01-10T17:56:41.871Z Sent 2 2 30 34
39852024-01-10T17:56:41.872Z Done 28 28 0 56
39862024-01-10T17:56:41.872Z Skipped 0 0 0 0
39872024-01-10T17:56:41.872Z Error 0 0 0 0
39882024-01-10T17:56:41.872ZLast Flush: 1010 1010 0
39892024-01-10T17:56:41.872ZDownstairs last five completed:
39902024-01-10T17:56:41.872ZUpstairs last five completed: 30 29 28 27 26
39912024-01-10T17:56:41.872ZWrote out file "/tmp/test_repair_verify.data"
39922024-01-10T17:56:41.876Z
39932024-01-10T17:56:41.879ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
39942024-01-10T17:56:41.909ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
39952024-01-10T17:56:41.909Z 0 000-019 32 32 32 62 62 62 F F F
39962024-01-10T17:56:41.909Z 1 020-039 35 35 35 66 66 66 F F F
39972024-01-10T17:56:41.909Z 2 040-059 32 32 32 62 62 62 F F F
39982024-01-10T17:56:41.909Z 3 060-079 33 33 33 64 64 64 F F F
39992024-01-10T17:56:41.909Z 4 080-099 35 35 35 66 66 66 F F F
40002024-01-10T17:56:41.909Z 5 100-119 28 28 28 54 54 54 F F F
40012024-01-10T17:56:41.909Z 6 120-139 30 30 30 57 57 57 F F F
40022024-01-10T17:56:41.910Z 7 140-159 32 32 32 61 61 61 T T F <---
40032024-01-10T17:56:41.910Z 8 160-179 28 28 28 54 54 54 T T F <---
40042024-01-10T17:56:41.910Z 9 180-199 28 28 28 54 54 54 T T F <---
40052024-01-10T17:56:41.923Z 10 200-219 34 34 34 65 65 65 F F F
40062024-01-10T17:56:41.923Z 11 220-239 20 20 20 39 39 39 T T F <---
40072024-01-10T17:56:41.923Z 12 240-259 32 32 32 62 62 62 F F F
40082024-01-10T17:56:41.923Z 13 260-279 34 34 34 65 65 65 T T F <---
40092024-01-10T17:56:41.923Z 14 280-299 36 36 35 69 69 66 F F F <---
40102024-01-10T17:56:41.924Z 15 300-319 36 36 32 68 68 60 T T F <---
40112024-01-10T17:56:41.924Z 16 320-339 33 33 33 64 64 64 T T F <---
40122024-01-10T17:56:41.924Z 17 340-359 34 34 34 65 65 65 F F F
40132024-01-10T17:56:41.924Z 18 360-379 32 32 32 61 61 61 T T F <---
40142024-01-10T17:56:41.924Z 19 380-399 30 30 30 56 56 56 F F F
40152024-01-10T17:56:41.924Z 20 400-419 35 35 35 66 66 66 T T F <---
40162024-01-10T17:56:41.924Z 21 420-439 35 35 35 66 66 66 F F F
40172024-01-10T17:56:41.925Z 22 440-459 34 34 34 65 65 65 F F F
40182024-01-10T17:56:41.925Z 23 460-479 36 36 28 68 68 54 F F F <---
40192024-01-10T17:56:41.925Z 24 480-499 32 32 32 62 62 62 T T F <---
40202024-01-10T17:56:41.925Z 25 500-519 36 36 32 67 67 63 T T F <---
40212024-01-10T17:56:41.925Z 26 520-539 35 35 35 66 66 66 F F F
40222024-01-10T17:56:41.925Z 27 540-559 28 28 28 54 54 54 T T F <---
40232024-01-10T17:56:41.926Z 28 560-579 35 35 35 66 66 66 T T F <---
40242024-01-10T17:56:41.926Z 29 580-599 30 30 30 57 57 57 F F F
40252024-01-10T17:56:41.926ZMax gen: 36, Max flush: 69
40262024-01-10T17:56:41.927ZError: Difference in extent metadata found!
40272024-01-10T17:56:41.927ZA Difference in extent metadata is expected here
40282024-01-10T17:56:41.927ZOn loop 18
40292024-01-10T17:56:41.927Z
40302024-01-10T17:56:41.933ZVerifying data now
40312024-01-10T17:56:47.366ZNo change after verify
40322024-01-10T17:56:47.366ZLoop: 18 Downstairs dump after verify (and repair):
40332024-01-10T17:56:47.388ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40342024-01-10T17:56:47.389Z 0 000-019 32 32 32 62 62 62 F F F
40352024-01-10T17:56:47.389Z 1 020-039 35 35 35 66 66 66 F F F
40362024-01-10T17:56:47.390Z 2 040-059 32 32 32 62 62 62 F F F
40372024-01-10T17:56:47.390Z 3 060-079 33 33 33 64 64 64 F F F
40382024-01-10T17:56:47.394Z 4 080-099 35 35 35 66 66 66 F F F
40392024-01-10T17:56:47.394Z 5 100-119 28 28 28 54 54 54 F F F
40402024-01-10T17:56:47.395Z 6 120-139 30 30 30 57 57 57 F F F
40412024-01-10T17:56:47.395Z 7 140-159 37 37 37 70 70 70 F F F
40422024-01-10T17:56:47.395Z 8 160-179 37 37 37 70 70 70 F F F
40432024-01-10T17:56:47.395Z 9 180-199 37 37 37 70 70 70 F F F
40442024-01-10T17:56:47.395Z 10 200-219 34 34 34 65 65 65 F F F
40452024-01-10T17:56:47.395Z 11 220-239 37 37 37 70 70 70 F F F
40462024-01-10T17:56:47.395Z 12 240-259 32 32 32 62 62 62 F F F
40472024-01-10T17:56:47.395Z 13 260-279 37 37 37 70 70 70 F F F
40482024-01-10T17:56:47.395Z 14 280-299 36 36 36 69 69 69 F F F
40492024-01-10T17:56:47.395Z 15 300-319 37 37 37 70 70 70 F F F
40502024-01-10T17:56:47.395Z 16 320-339 37 37 37 70 70 70 F F F
40512024-01-10T17:56:47.396Z 17 340-359 34 34 34 65 65 65 F F F
40522024-01-10T17:56:47.396Z 18 360-379 37 37 37 70 70 70 F F F
40532024-01-10T17:56:47.396Z 19 380-399 30 30 30 56 56 56 F F F
40542024-01-10T17:56:47.396Z 20 400-419 37 37 37 70 70 70 F F F
40552024-01-10T17:56:47.396Z 21 420-439 35 35 35 66 66 66 F F F
40562024-01-10T17:56:47.396Z 22 440-459 34 34 34 65 65 65 F F F
40572024-01-10T17:56:47.396Z 23 460-479 36 36 36 68 68 68 F F F
40582024-01-10T17:56:47.396Z 24 480-499 37 37 37 70 70 70 F F F
40592024-01-10T17:56:47.396Z 25 500-519 37 37 37 70 70 70 F F F
40602024-01-10T17:56:47.396Z 26 520-539 35 35 35 66 66 66 F F F
40612024-01-10T17:56:47.397Z 27 540-559 37 37 37 70 70 70 F F F
40622024-01-10T17:56:47.397Z 28 560-579 37 37 37 70 70 70 F F F
40632024-01-10T17:56:47.397Z 29 580-599 30 30 30 57 57 57 F F F
40642024-01-10T17:56:47.397ZMax gen: 37, Max flush: 70
40652024-01-10T17:56:47.397Z
40662024-01-10T17:56:47.397ZBegin loop 19
40672024-01-10T17:56:47.398ZDownstairs to restart: 0
40682024-01-10T17:56:47.423Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.422001207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307}
40692024-01-10T17:56:47.423Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.42256924Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307}
40702024-01-10T17:56:47.428Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.422633852Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307}
40712024-01-10T17:56:47.428Z{"msg":"Crucible 70011795-2bcc-4f12-bc26-f7f05d369838 has session id: aa12e56a-8580-4684-a6b3-bbba5bc21149","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.422690466Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40722024-01-10T17:56:47.428Z{"msg":"Upstairs opts: Upstairs UUID: 70011795-2bcc-4f12-bc26-f7f05d369838, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v"Crucible runtime is spawned
40732024-01-10T17:56:47.428Z:0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.422892213Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40742024-01-10T17:56:47.428Z{"msg":"Crucible stats registered with UUID: 70011795-2bcc-4f12-bc26-f7f05d369838","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.422958694Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40752024-01-10T17:56:47.428Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423024596Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307}
40762024-01-10T17:56:47.428Z{"msg":"70011795-2bcc-4f12-bc26-f7f05d369838 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423065726Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40772024-01-10T17:56:47.428Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423117125Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"0","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40782024-01-10T17:56:47.428Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423159364Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"0","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40792024-01-10T17:56:47.429Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423201065Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"1","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40802024-01-10T17:56:47.429Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423241862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"1","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40812024-01-10T17:56:47.429Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423280261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"2","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40822024-01-10T17:56:47.429Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423342847Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"2","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40832024-01-10T17:56:47.429Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423384621Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40842024-01-10T17:56:47.429Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423450813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40852024-01-10T17:56:47.429Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423553147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40862024-01-10T17:56:47.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423654715Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40872024-01-10T17:56:47.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423702396Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40882024-01-10T17:56:47.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.423950559Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40892024-01-10T17:56:47.430Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.424529483Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"1","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40902024-01-10T17:56:47.430Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.42462149Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"1","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40912024-01-10T17:56:47.430Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.424838402Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"2","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40922024-01-10T17:56:47.431Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.424923211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"2","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40932024-01-10T17:56:47.431Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.425595985Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"1","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40942024-01-10T17:56:47.431Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.425868498Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"2","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40952024-01-10T17:56:47.437Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.426356916Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"1","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40962024-01-10T17:56:47.440Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.426420663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"1","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40972024-01-10T17:56:47.440Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.426447076Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40982024-01-10T17:56:47.440Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.426674968Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"2","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
40992024-01-10T17:56:47.440Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.426761802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"2","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41002024-01-10T17:56:47.440Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.426835297Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41012024-01-10T17:56:47.441Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.426898526Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"0","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41022024-01-10T17:56:47.441Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.426926095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"0","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41032024-01-10T17:56:47.441Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.427668425Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"0","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41042024-01-10T17:56:47.441Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428312719Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"0","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41052024-01-10T17:56:47.441Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428380043Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"0","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41062024-01-10T17:56:47.442Z{"msg":"[0]R flush_numbers[0..12]: [62, 66, 62, 64, 66, 54, 57, 70, 70, 70, 65, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428508244Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41072024-01-10T17:56:47.442Z{"msg":"[0]R generation[0..12]: [32, 35, 32, 33, 35, 28, 30, 37, 37, 37, 34, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428536453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41082024-01-10T17:56:47.442ZWait for a query_work_queue command to finish before sending IO
41092024-01-10T17:56:47.443Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428601435Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41102024-01-10T17:56:47.443Z{"msg":"[1]R flush_numbers[0..12]: [62, 66, 62, 64, 66, 54, 57, 70, 70, 70, 65, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428628594Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41112024-01-10T17:56:47.443Z{"msg":"[1]R generation[0..12]: [32, 35, 32, 33, 35, 28, 30, 37, 37, 37, 34, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428653879Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41122024-01-10T17:56:47.443Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428701076Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41132024-01-10T17:56:47.443Z{"msg":"[2]R flush_numbers[0..12]: [62, 66, 62, 64, 66, 54, 57, 70, 70, 70, 65, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428726362Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41142024-01-10T17:56:47.443Z{"msg":"[2]R generation[0..12]: [32, 35, 32, 33, 35, 28, 30, 37, 37, 37, 34, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.42877542Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41152024-01-10T17:56:47.443Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.42880895Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41162024-01-10T17:56:47.445Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.42883198Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41172024-01-10T17:56:47.445Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428853908Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41182024-01-10T17:56:47.445Z{"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428897419Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41192024-01-10T17:56:47.445Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428920458Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"":"downstairs","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41202024-01-10T17:56:47.445Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428942137Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41212024-01-10T17:56:47.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.428979765Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"0","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41222024-01-10T17:56:47.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.429002742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"1","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41232024-01-10T17:56:47.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.42902422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"client":"2","":"downstairs","session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41242024-01-10T17:56:47.446Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.42904589Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41252024-01-10T17:56:47.446Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.429089155Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41262024-01-10T17:56:47.446Z{"msg":"70011795-2bcc-4f12-bc26-f7f05d369838 is now active with session: aa12e56a-8580-4684-a6b3-bbba5bc21149","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.429119167Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41272024-01-10T17:56:47.447Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.429172706Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307,"session_id":"aa12e56a-8580-4684-a6b3-bbba5bc21149"}
41282024-01-10T17:56:47.447Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:47.429199744Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1307}
41292024-01-10T17:56:47.447ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41302024-01-10T17:56:47.447ZLoading write count information from file "/tmp/test_repair_verify.data"
41312024-01-10T17:56:47.447ZRun Repair workload
41322024-01-10T17:56:47.447Z01/30 Write block 122 len 20480 data: 4 3 3 3 2
41332024-01-10T17:56:47.447Z02/30 Write block 39 len 32768 data: 2 3 4 4 3 4 4 3
41342024-01-10T17:56:47.451Z03/30 Write block 171 len 40960 data: 4 3 3 3 3 3 5 5 4 6
41352024-01-10T17:56:47.451Z04/30 Read block 110 len 12288
41362024-01-10T17:56:47.451Z05/30 Write block 119 len 36864 data: 4 5 5 5 4 4 4 3 2
41372024-01-10T17:56:47.451Z06/30 Read block 310 len 24576
41382024-01-10T17:56:47.451Z07/30 Read block 129 len 20480
41392024-01-10T17:56:47.452Z08/30 Read block 233 len 40960
41402024-01-10T17:56:47.452Z09/30 Read block 244 len 28672
41412024-01-10T17:56:47.455Z10/30 Write block 450 len 36864 data: 5 3 3 3 2 2 2 2 2
41422024-01-10T17:56:47.455Z11/30 Read block 90 len 4096
41432024-01-10T17:56:47.455Z12/30 Read block 82 len 32768
41442024-01-10T17:56:47.459Z13/30 Write block 244 len 8192 data: 4 4
41452024-01-10T17:56:47.459Z14/30 Read block 4 len 20480
41462024-01-10T17:56:47.463Z15/30 Write block 283 len 32768 data: 4 4 4 2 2 2 4 6
41472024-01-10T17:56:47.463Z16/30 Write block 347 len 16384 data: 3 3 3 3
41482024-01-10T17:56:47.463Z17/30 Write block 97 len 24576 data: 5 5 5 4 3 3
41492024-01-10T17:56:47.466Z18/30 Write block 399 len 4096 data: 4
41502024-01-10T17:56:47.466Z19/30 Read block 456 len 28672
41512024-01-10T17:56:47.470Z20/30 Read block 131 len 16384
41522024-01-10T17:56:47.470Z21/30 Write block 562 len 16384 data: 4 5 5 6
41532024-01-10T17:56:47.470Z22/30 Read block 237 len 28672
41542024-01-10T17:56:47.474Z23/30 Read block 122 len 28672
41552024-01-10T17:56:47.477Z24/30 Read block 448 len 20480
41562024-01-10T17:56:47.477Z25/30 Write block 399 len 4096 data: 5
41572024-01-10T17:56:47.477Z26/30 Flush
41582024-01-10T17:56:47.483Z27/30 Write block 370 len 28672 data: 4 5 4 3 3 3 3
41592024-01-10T17:56:47.487Z28/30 Write block 514 len 32768 data: 7 5 6 6 5 4 4 4
41602024-01-10T17:56:47.487Z29/30 Read block 387 len 32768
41612024-01-10T17:56:47.491Z30/30 Read block 426 len 28672
41622024-01-10T17:56:47.495Z----------------------------------------------------------------
41632024-01-10T17:56:47.495Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
41642024-01-10T17:56:47.495ZStates: Active Active Active
41652024-01-10T17:56:47.495ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
41662024-01-10T17:56:47.495Z 1 Acked 1000 Write 5 Sent Done Done false
41672024-01-10T17:56:47.495Z 2 Acked 1001 Write 8 Sent Done Done false
41682024-01-10T17:56:47.495Z 3 Acked 1002 Write 10 Sent Done Done false
41692024-01-10T17:56:47.495Z 4 Acked 1003 Read 3 Sent Done Done false
41702024-01-10T17:56:47.495Z 5 Acked 1004 Write 9 Sent Done Done false
41712024-01-10T17:56:47.496Z 6 Acked 1005 Read 6 Sent Done Done false
41722024-01-10T17:56:47.496Z 7 Acked 1006 Read 5 Sent Done Done false
41732024-01-10T17:56:47.496Z 8 Acked 1007 Read 10 Sent Done Done false
41742024-01-10T17:56:47.496Z 9 Acked 1008 Read 7 Sent Done Done false
41752024-01-10T17:56:47.497Z 10 Acked 1009 Write 9 Sent Done Done false
41762024-01-10T17:56:47.497Z 11 Acked 1010 Read 1 Sent Done Done false
41772024-01-10T17:56:47.497Z 12 Acked 1011 Read 8 Sent Done Done false
41782024-01-10T17:56:47.497Z 13 Acked 1012 Write 2 Sent Done Done false
41792024-01-10T17:56:47.497Z 14 Acked 1013 Read 5 Sent Done Done false
41802024-01-10T17:56:47.497Z 15 Acked 1014 Write 8 Sent Done Done false
41812024-01-10T17:56:47.497Z 16 Acked 1015 Write 4 Sent Done Done false
41822024-01-10T17:56:47.497Z 17 Acked 1016 Write 6 Sent Done Done false
41832024-01-10T17:56:47.497Z 18 Acked 1017 Write 1 Sent Done Done false
41842024-01-10T17:56:47.497Z 19 Acked 1018 Read 7 Sent Done Done false
41852024-01-10T17:56:47.497Z 20 Acked 1019 Read 4 Sent Done Done false
41862024-01-10T17:56:47.497Z 21 Acked 1020 Write 4 Sent Done Done false
41872024-01-10T17:56:47.498Z 22 Acked 1021 Read 7 Sent Done Done false
41882024-01-10T17:56:47.498Z 23 Acked 1022 Read 7 Sent Done Done false
41892024-01-10T17:56:47.498Z 24 Acked 1023 Read 5 Sent Done Done false
41902024-01-10T17:56:47.498Z 25 Acked 1024 Write 1 Sent Done Done false
41912024-01-10T17:56:47.498Z 26 Acked 1025 Flush 0 Sent Done Done false
41922024-01-10T17:56:47.498Z 27 Acked 1026 Write 7 Sent Done Done false
41932024-01-10T17:56:47.498Z 28 Acked 1027 Write 8 Sent Done Done false
41942024-01-10T17:56:47.499Z 29 Acked 1028 Read 8 Sent Done Done false
41952024-01-10T17:56:47.499Z 30 Acked 1029 Read 7 Sent Sent Done false
41962024-01-10T17:56:47.499Z STATES DS:0 DS:1 DS:2 TOTAL
41972024-01-10T17:56:47.499Z New 0 0 0 0
41982024-01-10T17:56:47.499Z Sent 30 1 0 31
41992024-01-10T17:56:47.500Z Done 0 29 30 59
42002024-01-10T17:56:47.500Z Skipped 0 0 0 0
42012024-01-10T17:56:47.500Z Error 0 0 0 0
42022024-01-10T17:56:47.500ZLast Flush: 0 1025 1025
42032024-01-10T17:56:47.500ZDownstairs last five completed:
42042024-01-10T17:56:47.500ZUpstairs last five completed: 30 29 28 27 26
42052024-01-10T17:56:47.500ZWrote out file "/tmp/test_repair_verify.data"
42062024-01-10T17:56:47.504Z
42072024-01-10T17:56:47.508ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42082024-01-10T17:56:47.528ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42092024-01-10T17:56:47.529Z 0 000-019 32 32 32 62 62 62 F F F
42102024-01-10T17:56:47.529Z 1 020-039 35 38 38 66 71 71 F F F <---
42112024-01-10T17:56:47.529Z 2 040-059 32 38 38 62 71 71 F F F <---
42122024-01-10T17:56:47.529Z 3 060-079 33 33 33 64 64 64 F F F
42132024-01-10T17:56:47.533Z 4 080-099 35 38 38 66 71 71 F F F <---
42142024-01-10T17:56:47.533Z 5 100-119 28 38 38 54 71 71 F F F <---
42152024-01-10T17:56:47.534Z 6 120-139 30 38 38 57 71 71 F F F <---
42162024-01-10T17:56:47.534Z 7 140-159 37 37 37 70 70 70 F F F
42172024-01-10T17:56:47.534Z 8 160-179 37 38 38 70 71 71 F F F <---
42182024-01-10T17:56:47.534Z 9 180-199 37 38 38 70 71 71 F F F <---
42192024-01-10T17:56:47.535Z 10 200-219 34 34 34 65 65 65 F F F
42202024-01-10T17:56:47.535Z 11 220-239 37 37 37 70 70 70 F F F
42212024-01-10T17:56:47.535Z 12 240-259 32 38 38 62 71 71 F F F <---
42222024-01-10T17:56:47.535Z 13 260-279 37 37 37 70 70 70 F F F
42232024-01-10T17:56:47.535Z 14 280-299 36 38 38 69 71 71 F F F <---
42242024-01-10T17:56:47.535Z 15 300-319 37 37 37 70 70 70 F F F
42252024-01-10T17:56:47.535Z 16 320-339 37 37 37 70 70 70 F F F
42262024-01-10T17:56:47.535Z 17 340-359 34 38 38 65 71 71 F F F <---
42272024-01-10T17:56:47.535Z 18 360-379 37 37 37 70 70 70 F T T <---
42282024-01-10T17:56:47.535Z 19 380-399 30 38 38 56 71 71 F F F <---
42292024-01-10T17:56:47.535Z 20 400-419 37 37 37 70 70 70 F F F
42302024-01-10T17:56:47.535Z 21 420-439 35 35 35 66 66 66 F F F
42312024-01-10T17:56:47.535Z 22 440-459 34 38 38 65 71 71 F F F <---
42322024-01-10T17:56:47.536Z 23 460-479 36 36 36 68 68 68 F F F
42332024-01-10T17:56:47.536Z 24 480-499 37 37 37 70 70 70 F F F
42342024-01-10T17:56:47.536Z 25 500-519 37 37 37 70 70 70 F T T <---
42352024-01-10T17:56:47.536Z 26 520-539 35 35 35 66 66 66 F T T <---
42362024-01-10T17:56:47.536Z 27 540-559 37 37 37 70 70 70 F F F
42372024-01-10T17:56:47.536Z 28 560-579 37 38 38 70 71 71 F F F <---
42382024-01-10T17:56:47.536Z 29 580-599 30 30 30 57 57 57 F F F
42392024-01-10T17:56:47.536ZMax gen: 38, Max flush: 71
42402024-01-10T17:56:47.536ZError: Difference in extent metadata found!
42412024-01-10T17:56:47.540ZA Difference in extent metadata is expected here
42422024-01-10T17:56:47.540ZOn loop 19
42432024-01-10T17:56:47.541Z
42442024-01-10T17:56:47.541ZVerifying data now
42452024-01-10T17:56:53.025ZNo change after verify
42462024-01-10T17:56:53.025ZLoop: 19 Downstairs dump after verify (and repair):
42472024-01-10T17:56:53.047ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42482024-01-10T17:56:53.047Z 0 000-019 32 32 32 62 62 62 F F F
42492024-01-10T17:56:53.047Z 1 020-039 38 38 38 71 71 71 F F F
42502024-01-10T17:56:53.048Z 2 040-059 38 38 38 71 71 71 F F F
42512024-01-10T17:56:53.048Z 3 060-079 33 33 33 64 64 64 F F F
42522024-01-10T17:56:53.048Z 4 080-099 38 38 38 71 71 71 F F F
42532024-01-10T17:56:53.048Z 5 100-119 38 38 38 71 71 71 F F F
42542024-01-10T17:56:53.048Z 6 120-139 38 38 38 71 71 71 F F F
42552024-01-10T17:56:53.051Z 7 140-159 37 37 37 70 70 70 F F F
42562024-01-10T17:56:53.051Z 8 160-179 38 38 38 71 71 71 F F F
42572024-01-10T17:56:53.051Z 9 180-199 38 38 38 71 71 71 F F F
42582024-01-10T17:56:53.051Z 10 200-219 34 34 34 65 65 65 F F F
42592024-01-10T17:56:53.051Z 11 220-239 37 37 37 70 70 70 F F F
42602024-01-10T17:56:53.051Z 12 240-259 38 38 38 71 71 71 F F F
42612024-01-10T17:56:53.051Z 13 260-279 37 37 37 70 70 70 F F F
42622024-01-10T17:56:53.051Z 14 280-299 38 38 38 71 71 71 F F F
42632024-01-10T17:56:53.052Z 15 300-319 37 37 37 70 70 70 F F F
42642024-01-10T17:56:53.052Z 16 320-339 37 37 37 70 70 70 F F F
42652024-01-10T17:56:53.053Z 17 340-359 38 38 38 71 71 71 F F F
42662024-01-10T17:56:53.053Z 18 360-379 39 39 39 72 72 72 F F F
42672024-01-10T17:56:53.053Z 19 380-399 38 38 38 71 71 71 F F F
42682024-01-10T17:56:53.053Z 20 400-419 37 37 37 70 70 70 F F F
42692024-01-10T17:56:53.053Z 21 420-439 35 35 35 66 66 66 F F F
42702024-01-10T17:56:53.053Z 22 440-459 38 38 38 71 71 71 F F F
42712024-01-10T17:56:53.053Z 23 460-479 36 36 36 68 68 68 F F F
42722024-01-10T17:56:53.053Z 24 480-499 37 37 37 70 70 70 F F F
42732024-01-10T17:56:53.053Z 25 500-519 39 39 39 72 72 72 F F F
42742024-01-10T17:56:53.053Z 26 520-539 39 39 39 72 72 72 F F F
42752024-01-10T17:56:53.053Z 27 540-559 37 37 37 70 70 70 F F F
42762024-01-10T17:56:53.053Z 28 560-579 38 38 38 71 71 71 F F F
42772024-01-10T17:56:53.053Z 29 580-599 30 30 30 57 57 57 F F F
42782024-01-10T17:56:53.053ZMax gen: 39, Max flush: 72
42792024-01-10T17:56:53.057Z
42802024-01-10T17:56:53.057ZBegin loop 20
42812024-01-10T17:56:53.057ZDownstairs to restart: 0
42822024-01-10T17:56:53.083Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.082386289Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315}
42832024-01-10T17:56:53.083Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.082924831Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315}
42842024-01-10T17:56:53.088Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083021188Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315}
42852024-01-10T17:56:53.089Z{"msg":"Crucible 074b9142-6fdf-4274-be1d-3c3f3da62bbb has session id: 6e03adf5-4500-4c4f-9468-932d5b2527d3","v":0,"name":"crucible","Crucible runtime is spawned
42862024-01-10T17:56:53.089Zlevel":30,"time":"2024-01-10T17:56:53.083098702Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42872024-01-10T17:56:53.089Z{"msg":"Upstairs opts: Upstairs UUID: 074b9142-6fdf-4274-be1d-3c3f3da62bbb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083178998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42882024-01-10T17:56:53.089Z{"msg":"Crucible stats registered with UUID: 074b9142-6fdf-4274-be1d-3c3f3da62bbb","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083255513Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42892024-01-10T17:56:53.089Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083297211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315}
42902024-01-10T17:56:53.089Z{"msg":"074b9142-6fdf-4274-be1d-3c3f3da62bbb active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083532325Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42912024-01-10T17:56:53.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083603362Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"0","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42922024-01-10T17:56:53.089Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083651526Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"0","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42932024-01-10T17:56:53.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083690833Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"1","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42942024-01-10T17:56:53.089Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083752407Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"1","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42952024-01-10T17:56:53.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083795977Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"2","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42962024-01-10T17:56:53.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083860782Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"2","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42972024-01-10T17:56:53.090Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083929679Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42982024-01-10T17:56:53.090Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.083974789Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
42992024-01-10T17:56:53.090Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.084096388Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43002024-01-10T17:56:53.090Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.084201221Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43012024-01-10T17:56:53.090Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.084572907Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43022024-01-10T17:56:53.091Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.084641938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43032024-01-10T17:56:53.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.085445255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"1","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43042024-01-10T17:56:53.091Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.085529558Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"1","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43052024-01-10T17:56:53.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.085579915Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"2","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43062024-01-10T17:56:53.091Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.085668541Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"2","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43072024-01-10T17:56:53.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.086378178Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"0","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43082024-01-10T17:56:53.092Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.086453471Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"0","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43092024-01-10T17:56:53.111Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.086500121Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"2","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43102024-01-10T17:56:53.111Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.086855475Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"1","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43112024-01-10T17:56:53.111Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.087515532Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"0","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43122024-01-10T17:56:53.111Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.087630105Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"2","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43132024-01-10T17:56:53.111Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.087715694Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"2","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43142024-01-10T17:56:53.111Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.087760641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43152024-01-10T17:56:53.111Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.087798328Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"1","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43162024-01-10T17:56:53.111Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.087839183Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"1","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43172024-01-10T17:56:53.111Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.087891691Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43182024-01-10T17:56:53.111Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.0883885Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"0","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43192024-01-10T17:56:53.111Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088460699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"0","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43202024-01-10T17:56:53.111Z{"msg":"[0]R flush_numbers[0..12]: [62, 71, 71, 64, 71, 71, 71, 70, 71, 71, 65, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088489789Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43212024-01-10T17:56:53.111Z{"msg":"[0]R generation[0..12]: [32, 38, 38, 33, 38, 38, 38, 37, 38, 38, 34, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088532523Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43222024-01-10T17:56:53.111Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.0885577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43232024-01-10T17:56:53.111Z{"msg":"[1]R flush_numbers[0..12]: [62, 71, 71, 64, 71, 71, 71, 70, 71, 71, 65, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088599667Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43242024-01-10T17:56:53.111Z{"msg":"[1]R generation[0..12]: [32, 38, 38, 33, 38, 38, 38, 37, 38, 38, 34, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088623637Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43252024-01-10T17:56:53.112Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088673883Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43262024-01-10T17:56:53.112Z{"msg":"[2]R flush_numbers[0..12]: [62, 71, 71, 64, 71, 71, 71, 70, 71, 71, 65, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088698089Z","hostname":"Wait for a query_work_queue command to finish before sending IO
43272024-01-10T17:56:53.112Zip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43282024-01-10T17:56:53.112Z{"msg":"[2]R generation[0..12]: [32, 38, 38, 33, 38, 38, 38, 37, 38, 38, 34, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088808791Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43292024-01-10T17:56:53.112Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088862561Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43302024-01-10T17:56:53.112Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088886415Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43312024-01-10T17:56:53.112Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088908389Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43322024-01-10T17:56:53.112Z{"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088947718Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43332024-01-10T17:56:53.112Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088970593Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"":"downstairs","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43342024-01-10T17:56:53.112Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.088992153Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43352024-01-10T17:56:53.113Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.089045861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"0","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43362024-01-10T17:56:53.113Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.089084401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"1","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43372024-01-10T17:56:53.113Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.089158619Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"client":"2","":"downstairs","session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43382024-01-10T17:56:53.113Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.089198695Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43392024-01-10T17:56:53.113Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.089256701Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43402024-01-10T17:56:53.113Z{"msg":"074b9142-6fdf-4274-be1d-3c3f3da62bbb is now active with session: 6e03adf5-4500-4c4f-9468-932d5b2527d3","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.089297882Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43412024-01-10T17:56:53.113Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.089349385Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315,"session_id":"6e03adf5-4500-4c4f-9468-932d5b2527d3"}
43422024-01-10T17:56:53.113Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:53.089388757Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1315}
43432024-01-10T17:56:53.113ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43442024-01-10T17:56:53.114ZLoading write count information from file "/tmp/test_repair_verify.data"
43452024-01-10T17:56:53.114ZRun Repair workload
43462024-01-10T17:56:53.114Z01/30 Write block 137 len 24576 data: 3 3 3 4 5 4
43472024-01-10T17:56:53.114Z02/30 Write block 64 len 32768 data: 5 6 7 4 4 4 4 2
43482024-01-10T17:56:53.115Z03/30 Read block 469 len 40960
43492024-01-10T17:56:53.115Z04/30 Read block 381 len 24576
43502024-01-10T17:56:53.115Z05/30 Read block 45 len 28672
43512024-01-10T17:56:53.118Z06/30 Write block 38 len 4096 data: 3
43522024-01-10T17:56:53.118Z07/30 Read block 496 len 24576
43532024-01-10T17:56:53.118Z08/30 Write block 313 len 16384 data: 2 3 4 5
43542024-01-10T17:56:53.118Z09/30 Flush
43552024-01-10T17:56:53.118Z10/30 Write block 137 len 8192 data: 4 4
43562024-01-10T17:56:53.118Z11/30 Write block 342 len 24576 data: 5 5 3 3 3 4
43572024-01-10T17:56:53.118Z12/30 Read block 464 len 32768
43582024-01-10T17:56:53.119Z13/30 Read block 301 len 20480
43592024-01-10T17:56:53.122Z14/30 Write block 547 len 32768 data: 6 6 6 5 4 6 5 5
43602024-01-10T17:56:53.122Z15/30 Read block 183 len 28672
43612024-01-10T17:56:53.126Z16/30 Read block 515 len 36864
43622024-01-10T17:56:53.129Z17/30 Read block 566 len 8192
43632024-01-10T17:56:53.129Z18/30 Read block 10 len 16384
43642024-01-10T17:56:53.129Z19/30 Read block 140 len 28672
43652024-01-10T17:56:53.134Z20/30 Write block 344 len 8192 data: 4 4
43662024-01-10T17:56:53.134Z21/30 Write block 573 len 12288 data: 3 5 5
43672024-01-10T17:56:53.137Z22/30 Read block 542 len 4096
43682024-01-10T17:56:53.137Z23/30 Flush
43692024-01-10T17:56:53.137Z24/30 Write block 173 len 16384 data: 4 4 4 4
43702024-01-10T17:56:53.140Z25/30 Write block 239 len 24576 data: 4 4 3 3 3 5
43712024-01-10T17:56:53.140Z26/30 Flush
43722024-01-10T17:56:53.143Z27/30 Write block 349 len 4096 data: 4
43732024-01-10T17:56:53.143Z28/30 Read block 537 len 28672
43742024-01-10T17:56:53.146Z29/30 Read block 253 len 20480
43752024-01-10T17:56:53.150Z30/30 Write block 594 len 8192 data: 6 5
43762024-01-10T17:56:53.150Z----------------------------------------------------------------
43772024-01-10T17:56:53.150Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
43782024-01-10T17:56:53.150ZStates: Active Active Active
43792024-01-10T17:56:53.151ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
43802024-01-10T17:56:53.151Z 1 Acked 1000 Write 6 Done Done Done false
43812024-01-10T17:56:53.151Z 2 Acked 1001 Write 8 Sent Done Done false
43822024-01-10T17:56:53.151Z 3 Acked 1002 Read 10 Sent Done Done false
43832024-01-10T17:56:53.151Z 4 Acked 1003 Read 6 Sent Done Done false
43842024-01-10T17:56:53.151Z 5 Acked 1004 Read 7 Sent Done Done false
43852024-01-10T17:56:53.151Z 6 Acked 1005 Write 1 Sent Done Done false
43862024-01-10T17:56:53.151Z 7 Acked 1006 Read 6 Sent Done Done false
43872024-01-10T17:56:53.151Z 8 Acked 1007 Write 4 Sent Done Done false
43882024-01-10T17:56:53.151Z 9 Acked 1008 Flush 0 Sent Done Done false
43892024-01-10T17:56:53.151Z 10 Acked 1009 Write 2 Sent Done Done false
43902024-01-10T17:56:53.151Z 11 Acked 1010 Write 6 Sent Done Done false
43912024-01-10T17:56:53.151Z 12 Acked 1011 Read 8 Sent Done Done false
43922024-01-10T17:56:53.151Z 13 Acked 1012 Read 5 Sent Done Done false
43932024-01-10T17:56:53.151Z 14 Acked 1013 Write 8 Sent Done Done false
43942024-01-10T17:56:53.151Z 15 Acked 1014 Read 7 Sent Done Done false
43952024-01-10T17:56:53.154Z 16 Acked 1015 Read 9 Sent Done Done false
43962024-01-10T17:56:53.154Z 17 Acked 1016 Read 2 Sent Done Done false
43972024-01-10T17:56:53.154Z 18 Acked 1017 Read 4 Sent Done Done false
43982024-01-10T17:56:53.154Z 19 Acked 1018 Read 7 Sent Done Done false
43992024-01-10T17:56:53.154Z 20 Acked 1019 Write 2 Sent Done Done false
44002024-01-10T17:56:53.154Z 21 Acked 1020 Write 3 Sent Done Done false
44012024-01-10T17:56:53.154Z 22 Acked 1021 Read 1 Sent Done Done false
44022024-01-10T17:56:53.156Z 23 Acked 1022 Flush 0 Sent Done Done false
44032024-01-10T17:56:53.156Z 24 Acked 1023 Write 4 Sent Done Done false
44042024-01-10T17:56:53.156Z 25 Acked 1024 Write 6 Sent Done Done false
44052024-01-10T17:56:53.156Z 26 Acked 1025 Flush 0 Sent Done Done false
44062024-01-10T17:56:53.156Z 27 Acked 1026 Write 1 Sent Done Done false
44072024-01-10T17:56:53.156Z 28 Acked 1027 Read 7 Sent Done Done false
44082024-01-10T17:56:53.156Z 29 Acked 1028 Read 5 Sent Done Done false
44092024-01-10T17:56:53.156Z 30 Acked 1029 Write 2 Sent Sent Sent false
44102024-01-10T17:56:53.156Z STATES DS:0 DS:1 DS:2 TOTAL
44112024-01-10T17:56:53.157Z New 0 0 0 0
44122024-01-10T17:56:53.157Z Sent 29 1 1 31
44132024-01-10T17:56:53.157Z Done 1 29 29 59
44142024-01-10T17:56:53.157Z Skipped 0 0 0 0
44152024-01-10T17:56:53.157Z Error 0 0 0 0
44162024-01-10T17:56:53.157ZLast Flush: 0 1025 1025
44172024-01-10T17:56:53.157ZDownstairs last five completed:
44182024-01-10T17:56:53.157ZUpstairs last five completed: 30 29 28 27 26
44192024-01-10T17:56:53.157ZWrote out file "/tmp/test_repair_verify.data"
44202024-01-10T17:56:53.163Z
44212024-01-10T17:56:53.167ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44222024-01-10T17:56:53.186ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44232024-01-10T17:56:53.186Z 0 000-019 32 32 32 62 62 62 F F F
44242024-01-10T17:56:53.186Z 1 020-039 38 40 40 71 73 73 F F F <---
44252024-01-10T17:56:53.186Z 2 040-059 38 38 38 71 71 71 F F F
44262024-01-10T17:56:53.186Z 3 060-079 33 40 40 64 73 73 F F F <---
44272024-01-10T17:56:53.186Z 4 080-099 38 38 38 71 71 71 F F F
44282024-01-10T17:56:53.189Z 5 100-119 38 38 38 71 71 71 F F F
44292024-01-10T17:56:53.190Z 6 120-139 38 40 40 71 74 74 T F F <---
44302024-01-10T17:56:53.190Z 7 140-159 37 40 40 70 73 73 T F F <---
44312024-01-10T17:56:53.190Z 8 160-179 38 40 40 71 75 75 F F F <---
44322024-01-10T17:56:53.190Z 9 180-199 38 38 38 71 71 71 F F F
44332024-01-10T17:56:53.190Z 10 200-219 34 34 34 65 65 65 F F F
44342024-01-10T17:56:53.190Z 11 220-239 37 40 40 70 75 75 F F F <---
44352024-01-10T17:56:53.190Z 12 240-259 38 40 40 71 75 75 F F F <---
44362024-01-10T17:56:53.190Z 13 260-279 37 37 37 70 70 70 F F F
44372024-01-10T17:56:53.190Z 14 280-299 38 38 38 71 71 71 F F F
44382024-01-10T17:56:53.190Z 15 300-319 37 40 40 70 73 73 F F F <---
44392024-01-10T17:56:53.190Z 16 320-339 37 37 37 70 70 70 F F F
44402024-01-10T17:56:53.190Z 17 340-359 38 40 40 71 74 74 F T T <---
44412024-01-10T17:56:53.190Z 18 360-379 39 39 39 72 72 72 F F F
44422024-01-10T17:56:53.190Z 19 380-399 38 38 38 71 71 71 F F F
44432024-01-10T17:56:53.190Z 20 400-419 37 37 37 70 70 70 F F F
44442024-01-10T17:56:53.190Z 21 420-439 35 35 35 66 66 66 F F F
44452024-01-10T17:56:53.190Z 22 440-459 38 38 38 71 71 71 F F F
44462024-01-10T17:56:53.190Z 23 460-479 36 36 36 68 68 68 F F F
44472024-01-10T17:56:53.190Z 24 480-499 37 37 37 70 70 70 F F F
44482024-01-10T17:56:53.190Z 25 500-519 39 39 39 72 72 72 F F F
44492024-01-10T17:56:53.190Z 26 520-539 39 39 39 72 72 72 F F F
44502024-01-10T17:56:53.190Z 27 540-559 37 40 40 70 74 74 F F F <---
44512024-01-10T17:56:53.190Z 28 560-579 38 40 40 71 74 74 F F F <---
44522024-01-10T17:56:53.190Z 29 580-599 30 30 30 57 57 57 F T T <---
44532024-01-10T17:56:53.191ZMax gen: 40, Max flush: 75
44542024-01-10T17:56:53.191ZError: Difference in extent metadata found!
44552024-01-10T17:56:53.194ZA Difference in extent metadata is expected here
44562024-01-10T17:56:53.194ZOn loop 20
44572024-01-10T17:56:53.194Z
44582024-01-10T17:56:53.197ZVerifying data now
44592024-01-10T17:56:59.398ZNo change after verify
44602024-01-10T17:56:59.398ZLoop: 20 Downstairs dump after verify (and repair):
44612024-01-10T17:56:59.420ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44622024-01-10T17:56:59.420Z 0 000-019 32 32 32 62 62 62 F F F
44632024-01-10T17:56:59.420Z 1 020-039 40 40 40 73 73 73 F F F
44642024-01-10T17:56:59.420Z 2 040-059 38 38 38 71 71 71 F F F
44652024-01-10T17:56:59.420Z 3 060-079 40 40 40 73 73 73 F F F
44662024-01-10T17:56:59.421Z 4 080-099 38 38 38 71 71 71 F F F
44672024-01-10T17:56:59.421Z 5 100-119 38 38 38 71 71 71 F F F
44682024-01-10T17:56:59.421Z 6 120-139 40 40 40 74 74 74 F F F
44692024-01-10T17:56:59.421Z 7 140-159 40 40 40 73 73 73 F F F
44702024-01-10T17:56:59.421Z 8 160-179 40 40 40 75 75 75 F F F
44712024-01-10T17:56:59.421Z 9 180-199 38 38 38 71 71 71 F F F
44722024-01-10T17:56:59.421Z 10 200-219 34 34 34 65 65 65 F F F
44732024-01-10T17:56:59.421Z 11 220-239 40 40 40 75 75 75 F F F
44742024-01-10T17:56:59.425Z 12 240-259 40 40 40 75 75 75 F F F
44752024-01-10T17:56:59.425Z 13 260-279 37 37 37 70 70 70 F F F
44762024-01-10T17:56:59.425Z 14 280-299 38 38 38 71 71 71 F F F
44772024-01-10T17:56:59.425Z 15 300-319 40 40 40 73 73 73 F F F
44782024-01-10T17:56:59.425Z 16 320-339 37 37 37 70 70 70 F F F
44792024-01-10T17:56:59.425Z 17 340-359 41 41 41 76 76 76 F F F
44802024-01-10T17:56:59.425Z 18 360-379 39 39 39 72 72 72 F F F
44812024-01-10T17:56:59.425Z 19 380-399 38 38 38 71 71 71 F F F
44822024-01-10T17:56:59.425Z 20 400-419 37 37 37 70 70 70 F F F
44832024-01-10T17:56:59.425Z 21 420-439 35 35 35 66 66 66 F F F
44842024-01-10T17:56:59.425Z 22 440-459 38 38 38 71 71 71 F F F
44852024-01-10T17:56:59.425Z 23 460-479 36 36 36 68 68 68 F F F
44862024-01-10T17:56:59.426Z 24 480-499 37 37 37 70 70 70 F F F
44872024-01-10T17:56:59.426Z 25 500-519 39 39 39 72 72 72 F F F
44882024-01-10T17:56:59.426Z 26 520-539 39 39 39 72 72 72 F F F
44892024-01-10T17:56:59.426Z 27 540-559 40 40 40 74 74 74 F F F
44902024-01-10T17:56:59.426Z 28 560-579 40 40 40 74 74 74 F F F
44912024-01-10T17:56:59.426Z 29 580-599 41 41 41 76 76 76 F F F
44922024-01-10T17:56:59.426ZMax gen: 41, Max flush: 76
44932024-01-10T17:56:59.429Z
44942024-01-10T17:56:59.429ZBegin loop 21
44952024-01-10T17:56:59.429ZDownstairs to restart: 2
44962024-01-10T17:56:59.460Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.456662027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323}
44972024-01-10T17:56:59.460Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.457166519Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323}
44982024-01-10T17:56:59.460Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.457196201Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323}
44992024-01-10T17:56:59.496Z{"msg":"Crucible e55174ac-e2da-47b3-8aed-1b25e76143fd has session id: 972be5c1-fddd-48ac-9b8b-928d3351d956","v":0,"name":"crucible","Crucible runtime is spawned
45002024-01-10T17:56:59.498Zlevel":30,"time":"2024-01-10T17:56:59.457450781Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45012024-01-10T17:56:59.498Z{"msg":"Upstairs opts: Upstairs UUID: e55174ac-e2da-47b3-8aed-1b25e76143fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.45755622Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45022024-01-10T17:56:59.498Z{"msg":"Crucible stats registered with UUID: e55174ac-e2da-47b3-8aed-1b25e76143fd","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.457593978Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45032024-01-10T17:56:59.498Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.457647343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323}
45042024-01-10T17:56:59.498Z{"msg":"e55174ac-e2da-47b3-8aed-1b25e76143fd active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.457669313Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45052024-01-10T17:56:59.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.457720407Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"0","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45062024-01-10T17:56:59.498Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.457753393Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"0","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45072024-01-10T17:56:59.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.457865432Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"1","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45082024-01-10T17:56:59.498Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.457910293Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"1","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45092024-01-10T17:56:59.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.457962114Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"2","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45102024-01-10T17:56:59.500Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.458005173Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"2","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45112024-01-10T17:56:59.500Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.458106271Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45122024-01-10T17:56:59.500Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.458210059Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45132024-01-10T17:56:59.500Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.458321143Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45142024-01-10T17:56:59.500Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.458466409Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45152024-01-10T17:56:59.500Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.458777385Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45162024-01-10T17:56:59.500Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.458853054Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45172024-01-10T17:56:59.500Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.459534521Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"0","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45182024-01-10T17:56:59.500Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.459630655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"0","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45192024-01-10T17:56:59.500Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.459860139Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"1","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45202024-01-10T17:56:59.501Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.459922797Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"1","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45212024-01-10T17:56:59.501Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.46050933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"0","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45222024-01-10T17:56:59.501Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.460721117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"1","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45232024-01-10T17:56:59.501Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.461245223Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"2","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45242024-01-10T17:56:59.501Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.461313715Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"2","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45252024-01-10T17:56:59.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.461338995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"0","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45262024-01-10T17:56:59.501Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.46136141Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"0","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45272024-01-10T17:56:59.501Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.461459689Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45282024-01-10T17:56:59.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.461700129Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"1","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45292024-01-10T17:56:59.501Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.461751896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"1","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45302024-01-10T17:56:59.501Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.46177661Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45312024-01-10T17:56:59.501Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.462351544Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"2","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45322024-01-10T17:56:59.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463102026Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"2","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45332024-01-10T17:56:59.501Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463181646Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"2","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45342024-01-10T17:56:59.501Z{"msg":"[0]R flush_numbers[0..12]: [62, 73, 71, 73, 71, 71, 74, 73, 75, 71, 65, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463209278Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45352024-01-10T17:56:59.501Z{"msg":"[0]R generation[0..12]: [32, 40, 38, 40, 38, 38, 40, 40, 40, 38, 34, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463232676Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45362024-01-10T17:56:59.502Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463276638Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45372024-01-10T17:56:59.502Z{"msg":"[1]R flush_numbers[0..12]: [62, 73, 71, 73, 71, 71, 74, 73, 75, 71, 65, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463302015Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45382024-01-10T17:56:59.502Z{"msg":"[1]R generation[0..12]: [32, 40, 38, 40, 38, 38, 40, 40, 40, 38, 34, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463343496Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45392024-01-10T17:56:59.502Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463368372Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45402024-01-10T17:56:59.503Z{Wait for a query_work_queue command to finish before sending IO
45412024-01-10T17:56:59.503Z"msg":"[2]R flush_numbers[0..12]: [62, 73, 71, 73, 71, 71, 74, 73, 75, 71, 65, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463414686Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45422024-01-10T17:56:59.503Z{"msg":"[2]R generation[0..12]: [32, 40, 38, 40, 38, 38, 40, 40, 40, 38, 34, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463439195Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45432024-01-10T17:56:59.507Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463462578Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45442024-01-10T17:56:59.507Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463503456Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45452024-01-10T17:56:59.507Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463527201Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45462024-01-10T17:56:59.507Z{"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463548008Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45472024-01-10T17:56:59.507Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463588873Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"":"downstairs","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45482024-01-10T17:56:59.507Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463612072Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45492024-01-10T17:56:59.507Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463631833Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"0","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45502024-01-10T17:56:59.508Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.46368396Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"1","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45512024-01-10T17:56:59.508Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463756117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"client":"2","":"downstairs","session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45522024-01-10T17:56:59.508Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463784067Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45532024-01-10T17:56:59.508Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463813014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45542024-01-10T17:56:59.508Z{"msg":"e55174ac-e2da-47b3-8aed-1b25e76143fd is now active with session: 972be5c1-fddd-48ac-9b8b-928d3351d956","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463851469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45552024-01-10T17:56:59.508Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463883579Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323}
45562024-01-10T17:56:59.508Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:56:59.463942126Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1323,"session_id":"972be5c1-fddd-48ac-9b8b-928d3351d956"}
45572024-01-10T17:56:59.508ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45582024-01-10T17:56:59.508ZLoading write count information from file "/tmp/test_repair_verify.data"
45592024-01-10T17:56:59.508ZRun Repair workload
45602024-01-10T17:56:59.508Z01/30 Write block 269 len 4096 data: 5
45612024-01-10T17:56:59.508Z02/30 Read block 421 len 4096
45622024-01-10T17:56:59.508Z03/30 Read block 328 len 16384
45632024-01-10T17:56:59.508Z04/30 Read block 360 len 40960
45642024-01-10T17:56:59.508Z05/30 Read block 148 len 32768
45652024-01-10T17:56:59.509Z06/30 Write block 419 len 24576 data: 8 8 8 7 6 6
45662024-01-10T17:56:59.509Z07/30 Write block 186 len 40960 data: 5 4 5 5 5 4 4 4 5 5
45672024-01-10T17:56:59.509Z08/30 Read block 431 len 8192
45682024-01-10T17:56:59.509Z09/30 Write block 379 len 32768 data: 3 4 4 4 3 3 3 2
45692024-01-10T17:56:59.509Z10/30 Write block 130 len 24576 data: 3 2 2 2 3 3
45702024-01-10T17:56:59.510Z11/30 Read block 21 len 4096
45712024-01-10T17:56:59.510Z12/30 Flush
45722024-01-10T17:56:59.510Z13/30 Write block 368 len 24576 data: 3 3 5 6 5 4
45732024-01-10T17:56:59.510Z14/30 Read block 48 len 32768
45742024-01-10T17:56:59.510Z15/30 Read block 532 len 20480
45752024-01-10T17:56:59.510Z16/30 Read block 94 len 16384
45762024-01-10T17:56:59.510Z17/30 Read block 526 len 40960
45772024-01-10T17:56:59.510Z18/30 Flush
45782024-01-10T17:56:59.510Z19/30 Write block 139 len 12288 data: 4 5 6
45792024-01-10T17:56:59.510Z20/30 Flush
45802024-01-10T17:56:59.510Z21/30 Write block 351 len 12288 data: 3 3 3
45812024-01-10T17:56:59.516Z22/30 Read block 380 len 20480
45822024-01-10T17:56:59.516Z23/30 Write block 509 len 12288 data: 7 6 6
45832024-01-10T17:56:59.516Z24/30 Read block 371 len 32768
45842024-01-10T17:56:59.516Z25/30 Flush
45852024-01-10T17:56:59.516Z26/30 Write block 479 len 16384 data: 2 2 2 2
45862024-01-10T17:56:59.519Z27/30 Read block 29 len 28672
45872024-01-10T17:56:59.522Z28/30 Read block 482 len 28672
45882024-01-10T17:56:59.525Z29/30 Write block 165 len 24576 data: 8 7 6 5 4 3
45892024-01-10T17:56:59.525Z30/30 Read block 461 len 28672
45902024-01-10T17:56:59.529Z----------------------------------------------------------------
45912024-01-10T17:56:59.530Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
45922024-01-10T17:56:59.530ZStates: Active Active Active
45932024-01-10T17:56:59.530ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
45942024-01-10T17:56:59.530Z 1 Acked 1000 Write 1 Done Done Done false
45952024-01-10T17:56:59.530Z 2 Acked 1001 Read 1 Done Done Done false
45962024-01-10T17:56:59.531Z 3 Acked 1002 Read 4 Done Done Sent false
45972024-01-10T17:56:59.531Z 4 Acked 1003 Read 10 Done Done Sent false
45982024-01-10T17:56:59.531Z 5 Acked 1004 Read 8 Done Done Sent false
45992024-01-10T17:56:59.531Z 6 Acked 1005 Write 6 Done Done Sent false
46002024-01-10T17:56:59.531Z 7 Acked 1006 Write 10 Done Done Sent false
46012024-01-10T17:56:59.531Z 8 Acked 1007 Read 2 Done Done Sent false
46022024-01-10T17:56:59.531Z 9 Acked 1008 Write 8 Done Done Sent false
46032024-01-10T17:56:59.531Z 10 Acked 1009 Write 6 Done Done Sent false
46042024-01-10T17:56:59.531Z 11 Acked 1010 Read 1 Done Done Sent false
46052024-01-10T17:56:59.531Z 12 Acked 1011 Flush 0 Done Done Sent false
46062024-01-10T17:56:59.531Z 13 Acked 1012 Write 6 Done Done Sent false
46072024-01-10T17:56:59.531Z 14 Acked 1013 Read 8 Done Done Sent false
46082024-01-10T17:56:59.531Z 15 Acked 1014 Read 5 Done Done Sent false
46092024-01-10T17:56:59.531Z 16 Acked 1015 Read 4 Done Done Sent false
46102024-01-10T17:56:59.531Z 17 Acked 1016 Read 10 Done Done Sent false
46112024-01-10T17:56:59.531Z 18 Acked 1017 Flush 0 Done Done Sent false
46122024-01-10T17:56:59.531Z 19 Acked 1018 Write 3 Done Done Sent false
46132024-01-10T17:56:59.531Z 20 Acked 1019 Flush 0 Done Done Sent false
46142024-01-10T17:56:59.531Z 21 Acked 1020 Write 3 Done Done Sent false
46152024-01-10T17:56:59.531Z 22 Acked 1021 Read 5 Done Done Sent false
46162024-01-10T17:56:59.531Z 23 Acked 1022 Write 3 Done Done Sent false
46172024-01-10T17:56:59.531Z 24 Acked 1023 Read 8 Done Done Sent false
46182024-01-10T17:56:59.531Z 25 Acked 1024 Flush 0 Done Done Sent false
46192024-01-10T17:56:59.531Z 26 Acked 1025 Write 4 Done Done Sent false
46202024-01-10T17:56:59.531Z 27 Acked 1026 Read 7 Done Done Sent false
46212024-01-10T17:56:59.532Z 28 Acked 1027 Read 7 Done Done Sent false
46222024-01-10T17:56:59.532Z 29 Acked 1028 Write 6 Done Done Sent false
46232024-01-10T17:56:59.532Z 30 Acked 1029 Read 7 Done Sent Sent false
46242024-01-10T17:56:59.532Z STATES DS:0 DS:1 DS:2 TOTAL
46252024-01-10T17:56:59.532Z New 0 0 0 0
46262024-01-10T17:56:59.532Z Sent 0 1 28 29
46272024-01-10T17:56:59.532Z Done 30 29 2 61
46282024-01-10T17:56:59.532Z Skipped 0 0 0 0
46292024-01-10T17:56:59.532Z Error 0 0 0 0
46302024-01-10T17:56:59.532ZLast Flush: 1024 1024 0
46312024-01-10T17:56:59.532ZDownstairs last five completed:
46322024-01-10T17:56:59.532ZUpstairs last five completed: 30 29 28 27 26
46332024-01-10T17:56:59.532ZWrote out file "/tmp/test_repair_verify.data"
46342024-01-10T17:56:59.536Z
46352024-01-10T17:56:59.541ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46362024-01-10T17:56:59.566ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46372024-01-10T17:56:59.566Z 0 000-019 32 32 32 62 62 62 F F F
46382024-01-10T17:56:59.566Z 1 020-039 40 40 40 73 73 73 F F F
46392024-01-10T17:56:59.566Z 2 040-059 38 38 38 71 71 71 F F F
46402024-01-10T17:56:59.566Z 3 060-079 40 40 40 73 73 73 F F F
46412024-01-10T17:56:59.566Z 4 080-099 38 38 38 71 71 71 F F F
46422024-01-10T17:56:59.566Z 5 100-119 38 38 38 71 71 71 F F F
46432024-01-10T17:56:59.569Z 6 120-139 42 42 40 79 79 74 F F F <---
46442024-01-10T17:56:59.570Z 7 140-159 42 42 40 79 79 73 F F F <---
46452024-01-10T17:56:59.570Z 8 160-179 40 40 40 75 75 75 T T F <---
46462024-01-10T17:56:59.570Z 9 180-199 42 42 38 77 77 71 F F F <---
46472024-01-10T17:56:59.570Z 10 200-219 34 34 34 65 65 65 F F F
46482024-01-10T17:56:59.570Z 11 220-239 40 40 40 75 75 75 F F F
46492024-01-10T17:56:59.570Z 12 240-259 40 40 40 75 75 75 F F F
46502024-01-10T17:56:59.571Z 13 260-279 42 42 37 77 77 70 F F T <---
46512024-01-10T17:56:59.571Z 14 280-299 38 38 38 71 71 71 F F F
46522024-01-10T17:56:59.571Z 15 300-319 40 40 40 73 73 73 F F F
46532024-01-10T17:56:59.571Z 16 320-339 37 37 37 70 70 70 F F F
46542024-01-10T17:56:59.571Z 17 340-359 42 42 41 80 80 76 F F F <---
46552024-01-10T17:56:59.571Z 18 360-379 42 42 39 78 78 72 F F F <---
46562024-01-10T17:56:59.571Z 19 380-399 42 42 38 77 77 71 F F F <---
46572024-01-10T17:56:59.571Z 20 400-419 42 42 37 77 77 70 F F F <---
46582024-01-10T17:56:59.571Z 21 420-439 42 42 35 77 77 66 F F F <---
46592024-01-10T17:56:59.571Z 22 440-459 38 38 38 71 71 71 F F F
46602024-01-10T17:56:59.571Z 23 460-479 36 36 36 68 68 68 T T F <---
46612024-01-10T17:56:59.571Z 24 480-499 37 37 37 70 70 70 T T F <---
46622024-01-10T17:56:59.571Z 25 500-519 42 42 39 80 80 72 F F F <---
46632024-01-10T17:56:59.571Z 26 520-539 39 39 39 72 72 72 F F F
46642024-01-10T17:56:59.571Z 27 540-559 40 40 40 74 74 74 F F F
46652024-01-10T17:56:59.571Z 28 560-579 40 40 40 74 74 74 F F F
46662024-01-10T17:56:59.571Z 29 580-599 41 41 41 76 76 76 F F F
46672024-01-10T17:56:59.571ZMax gen: 42, Max flush: 80
46682024-01-10T17:56:59.571ZError: Difference in extent metadata found!
46692024-01-10T17:56:59.575ZA Difference in extent metadata is expected here
46702024-01-10T17:56:59.575ZOn loop 21
46712024-01-10T17:56:59.575Z
46722024-01-10T17:56:59.575ZVerifying data now
46732024-01-10T17:57:04.860ZNo change after verify
46742024-01-10T17:57:04.861ZLoop: 21 Downstairs dump after verify (and repair):
46752024-01-10T17:57:04.886ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46762024-01-10T17:57:04.886Z 0 000-019 32 32 32 62 62 62 F F F
46772024-01-10T17:57:04.886Z 1 020-039 40 40 40 73 73 73 F F F
46782024-01-10T17:57:04.887Z 2 040-059 38 38 38 71 71 71 F F F
46792024-01-10T17:57:04.887Z 3 060-079 40 40 40 73 73 73 F F F
46802024-01-10T17:57:04.887Z 4 080-099 38 38 38 71 71 71 F F F
46812024-01-10T17:57:04.887Z 5 100-119 38 38 38 71 71 71 F F F
46822024-01-10T17:57:04.892Z 6 120-139 42 42 42 79 79 79 F F F
46832024-01-10T17:57:04.892Z 7 140-159 42 42 42 79 79 79 F F F
46842024-01-10T17:57:04.892Z 8 160-179 43 43 43 81 81 81 F F F
46852024-01-10T17:57:04.892Z 9 180-199 42 42 42 77 77 77 F F F
46862024-01-10T17:57:04.892Z 10 200-219 34 34 34 65 65 65 F F F
46872024-01-10T17:57:04.892Z 11 220-239 40 40 40 75 75 75 F F F
46882024-01-10T17:57:04.892Z 12 240-259 40 40 40 75 75 75 F F F
46892024-01-10T17:57:04.892Z 13 260-279 42 42 42 77 77 77 F F F
46902024-01-10T17:57:04.892Z 14 280-299 38 38 38 71 71 71 F F F
46912024-01-10T17:57:04.892Z 15 300-319 40 40 40 73 73 73 F F F
46922024-01-10T17:57:04.893Z 16 320-339 37 37 37 70 70 70 F F F
46932024-01-10T17:57:04.893Z 17 340-359 42 42 42 80 80 80 F F F
46942024-01-10T17:57:04.893Z 18 360-379 42 42 42 78 78 78 F F F
46952024-01-10T17:57:04.893Z 19 380-399 42 42 42 77 77 77 F F F
46962024-01-10T17:57:04.893Z 20 400-419 42 42 42 77 77 77 F F F
46972024-01-10T17:57:04.893Z 21 420-439 42 42 42 77 77 77 F F F
46982024-01-10T17:57:04.893Z 22 440-459 38 38 38 71 71 71 F F F
46992024-01-10T17:57:04.894Z 23 460-479 43 43 43 81 81 81 F F F
47002024-01-10T17:57:04.894Z 24 480-499 43 43 43 81 81 81 F F F
47012024-01-10T17:57:04.894Z 25 500-519 42 42 42 80 80 80 F F F
47022024-01-10T17:57:04.894Z 26 520-539 39 39 39 72 72 72 F F F
47032024-01-10T17:57:04.894Z 27 540-559 40 40 40 74 74 74 F F F
47042024-01-10T17:57:04.894Z 28 560-579 40 40 40 74 74 74 F F F
47052024-01-10T17:57:04.894Z 29 580-599 41 41 41 76 76 76 F F F
47062024-01-10T17:57:04.894ZMax gen: 43, Max flush: 81
47072024-01-10T17:57:04.894Z
47082024-01-10T17:57:04.894ZBegin loop 22
47092024-01-10T17:57:04.894ZDownstairs to restart: 1
47102024-01-10T17:57:04.919Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.917557219Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331}
47112024-01-10T17:57:04.942Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.91802466Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331}
47122024-01-10T17:57:04.942Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.918131927Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331}
47132024-01-10T17:57:04.942Z{"msg":"Crucible 98ee0c33-c358-4052-9672-1e88dd26f3b2 has session id: d6ac8719-5ca2-4092-a818-0e855e9ad594","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.918166018Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47142024-01-10T17:57:04.942Z{"msg":"Upstairs opts: Upstairs UUID: 98ee0c33-c358-4052-9672-1e88dd26f3b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucibleCrucible runtime is spawned
47152024-01-10T17:57:04.942Z","level":30,"time":"2024-01-10T17:57:04.918245988Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47162024-01-10T17:57:04.942Z{"msg":"Crucible stats registered with UUID: 98ee0c33-c358-4052-9672-1e88dd26f3b2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.918332878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47172024-01-10T17:57:04.942Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.91845699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331}
47182024-01-10T17:57:04.942Z{"msg":"98ee0c33-c358-4052-9672-1e88dd26f3b2 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.918603142Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47192024-01-10T17:57:04.942Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.918665253Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"0","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47202024-01-10T17:57:04.942Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.918690883Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"0","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47212024-01-10T17:57:04.943Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.918713284Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"1","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47222024-01-10T17:57:04.943Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.918754394Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"1","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47232024-01-10T17:57:04.943Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.918783031Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"2","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47242024-01-10T17:57:04.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.918860089Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"2","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47252024-01-10T17:57:04.944Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.91888798Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47262024-01-10T17:57:04.944Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.919040666Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47272024-01-10T17:57:04.944Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.919126553Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47282024-01-10T17:57:04.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.919171373Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47292024-01-10T17:57:04.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.919213556Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47302024-01-10T17:57:04.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.91926879Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47312024-01-10T17:57:04.944Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.920732446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"2","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47322024-01-10T17:57:04.944Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.920821795Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"2","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47332024-01-10T17:57:04.944Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.920870712Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"0","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47342024-01-10T17:57:04.945Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.920975577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"0","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47352024-01-10T17:57:04.945Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.921004227Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"1","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47362024-01-10T17:57:04.945Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.921153935Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"1","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47372024-01-10T17:57:04.945Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.921660152Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"2","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47382024-01-10T17:57:04.946Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.921841534Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"0","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47392024-01-10T17:57:04.946Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.922246292Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"1","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47402024-01-10T17:57:04.946Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.922689095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"2","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47412024-01-10T17:57:04.946Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.922803497Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"2","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47422024-01-10T17:57:04.946Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.922881615Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47432024-01-10T17:57:04.946Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.922924775Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"0","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47442024-01-10T17:57:04.946Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.92310865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"0","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47452024-01-10T17:57:04.946Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923173912Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47462024-01-10T17:57:04.946Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923198558Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"1","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47472024-01-10T17:57:04.946Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923221027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"1","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47482024-01-10T17:57:04.946Z{"msg":"[0]R flush_numbers[0..12]: [62, 73, 71, 73, 71, 71, 79, 79, 81, 77, 65, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.9232632Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47492024-01-10T17:57:04.946Z{"msg":"[0]R generation[0..12]: [32, 40, 38, 40, 38, 38, 42, 42, 43, 42, 34, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923287988Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47502024-01-10T17:57:04.946Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923311787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47512024-01-10T17:57:04.946ZWait for a query_work_queue command to finish before sending IO
47522024-01-10T17:57:04.947Z{"msg":"[1]R flush_numbers[0..12]: [62, 73, 71, 73, 71, 71, 79, 79, 81, 77, 65, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923357436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47532024-01-10T17:57:04.947Z{"msg":"[1]R generation[0..12]: [32, 40, 38, 40, 38, 38, 42, 42, 43, 42, 34, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923381847Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47542024-01-10T17:57:04.947Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923416261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47552024-01-10T17:57:04.947Z{"msg":"[2]R flush_numbers[0..12]: [62, 73, 71, 73, 71, 71, 79, 79, 81, 77, 65, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923443981Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47562024-01-10T17:57:04.947Z{"msg":"[2]R generation[0..12]: [32, 40, 38, 40, 38, 38, 42, 42, 43, 42, 34, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923466725Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47572024-01-10T17:57:04.947Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923510014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47582024-01-10T17:57:04.947Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923533065Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47592024-01-10T17:57:04.947Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923556345Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47602024-01-10T17:57:04.947Z{"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923624987Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47612024-01-10T17:57:04.961Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923653333Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"":"downstairs","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47622024-01-10T17:57:04.961Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923702156Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47632024-01-10T17:57:04.961Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923724765Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"0","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47642024-01-10T17:57:04.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923746269Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"1","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47652024-01-10T17:57:04.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923785898Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"client":"2","":"downstairs","session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47662024-01-10T17:57:04.965Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923809072Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47672024-01-10T17:57:04.966Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923829159Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47682024-01-10T17:57:04.966Z{"msg":"98ee0c33-c358-4052-9672-1e88dd26f3b2 is now active with session: d6ac8719-5ca2-4092-a818-0e855e9ad594","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923851721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47692024-01-10T17:57:04.966Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923895553Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331,"session_id":"d6ac8719-5ca2-4092-a818-0e855e9ad594"}
47702024-01-10T17:57:04.966Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:04.923918088Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1331}
47712024-01-10T17:57:04.966ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
47722024-01-10T17:57:04.966ZLoading write count information from file "/tmp/test_repair_verify.data"
47732024-01-10T17:57:04.966ZRun Repair workload
47742024-01-10T17:57:04.966Z01/30 Write block 552 len 16384 data: 7 6 6 4
47752024-01-10T17:57:04.966Z02/30 Write block 547 len 36864 data: 7 7 7 6 5 8 7 7 5
47762024-01-10T17:57:04.966Z03/30 Write block 290 len 20480 data: 7 9 7 6 5
47772024-01-10T17:57:04.966Z04/30 Read block 585 len 8192
47782024-01-10T17:57:04.966Z05/30 Write block 30 len 16384 data: 7 5 5 4
47792024-01-10T17:57:04.966Z06/30 Read block 243 len 16384
47802024-01-10T17:57:04.966Z07/30 Read block 33 len 24576
47812024-01-10T17:57:04.966Z08/30 Read block 591 len 16384
47822024-01-10T17:57:04.967Z09/30 Write block 529 len 16384 data: 3 3 5 5
47832024-01-10T17:57:04.967Z10/30 Read block 403 len 20480
47842024-01-10T17:57:04.967Z11/30 Flush
47852024-01-10T17:57:04.967Z12/30 Write block 303 len 4096 data: 5
47862024-01-10T17:57:04.967Z13/30 Read block 307 len 20480
47872024-01-10T17:57:04.967Z14/30 Write block 131 len 24576 data: 3 3 3 4 4 2
47882024-01-10T17:57:04.967Z15/30 Write block 514 len 8192 data: 8 6
47892024-01-10T17:57:04.967Z16/30 Read block 279 len 28672
47902024-01-10T17:57:04.967Z17/30 Flush
47912024-01-10T17:57:04.973Z18/30 Write block 533 len 36864 data: 5 4 4 5 5 5 4 5 5
47922024-01-10T17:57:04.973Z19/30 Read block 465 len 20480
47932024-01-10T17:57:04.973Z20/30 Read block 136 len 20480
47942024-01-10T17:57:04.973Z21/30 Write block 586 len 20480 data: 4 3 3 4 4
47952024-01-10T17:57:04.973Z22/30 Write block 135 len 32768 data: 5 3 5 5 5 6 7 5
47962024-01-10T17:57:04.973Z23/30 Read block 513 len 20480
47972024-01-10T17:57:04.973Z24/30 Flush
47982024-01-10T17:57:04.974Z25/30 Write block 372 len 24576 data: 6 5 4 4 4 3
47992024-01-10T17:57:04.974Z26/30 Flush
48002024-01-10T17:57:04.979Z27/30 Write block 3 len 16384 data: 4 4 3 3
48012024-01-10T17:57:04.979Z28/30 Write block 102 len 8192 data: 4 3
48022024-01-10T17:57:04.979Z29/30 Write block 502 len 12288 data: 8 9 7
48032024-01-10T17:57:04.979Z30/30 Write block 166 len 36864 data: 8 7 6 5 4 5 4 5 5
48042024-01-10T17:57:04.979Z----------------------------------------------------------------
48052024-01-10T17:57:04.979Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48062024-01-10T17:57:04.979ZStates: Active Active Active
48072024-01-10T17:57:04.980ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48082024-01-10T17:57:04.980Z 1 Acked 1000 Write 4 Done Done Done false
48092024-01-10T17:57:04.980Z 2 Acked 1001 Write 9 Done Sent Done false
48102024-01-10T17:57:04.980Z 3 Acked 1002 Write 5 Done Sent Done false
48112024-01-10T17:57:04.980Z 4 Acked 1003 Read 2 Done Sent Done false
48122024-01-10T17:57:04.980Z 5 Acked 1004 Write 4 Done Sent Done false
48132024-01-10T17:57:04.980Z 6 Acked 1005 Read 4 Done Sent Done false
48142024-01-10T17:57:04.980Z 7 Acked 1006 Read 6 Done Sent Done false
48152024-01-10T17:57:04.980Z 8 Acked 1007 Read 4 Done Sent Done false
48162024-01-10T17:57:04.980Z 9 Acked 1008 Write 4 Done Sent Done false
48172024-01-10T17:57:04.980Z 10 Acked 1009 Read 5 Done Sent Done false
48182024-01-10T17:57:04.980Z 11 Acked 1010 Flush 0 Done Sent Done false
48192024-01-10T17:57:04.980Z 12 Acked 1011 Write 1 Done Sent Done false
48202024-01-10T17:57:04.980Z 13 Acked 1012 Read 5 Done Sent Done false
48212024-01-10T17:57:04.980Z 14 Acked 1013 Write 6 Done Sent Done false
48222024-01-10T17:57:04.980Z 15 Acked 1014 Write 2 Done Sent Done false
48232024-01-10T17:57:04.980Z 16 Acked 1015 Read 7 Done Sent Done false
48242024-01-10T17:57:04.980Z 17 Acked 1016 Flush 0 Done Sent Done false
48252024-01-10T17:57:04.980Z 18 Acked 1017 Write 9 Done Sent Done false
48262024-01-10T17:57:04.980Z 19 Acked 1018 Read 5 Done Sent Done false
48272024-01-10T17:57:04.982Z 20 Acked 1019 Read 5 Done Sent Done false
48282024-01-10T17:57:04.983Z 21 Acked 1020 Write 5 Done Sent Done false
48292024-01-10T17:57:04.983Z 22 Acked 1021 Write 8 Done Sent Done false
48302024-01-10T17:57:04.983Z 23 Acked 1022 Read 5 Done Sent Done false
48312024-01-10T17:57:04.983Z 24 Acked 1023 Flush 0 Done Sent Done false
48322024-01-10T17:57:04.983Z 25 Acked 1024 Write 6 Done Sent Done false
48332024-01-10T17:57:04.983Z 26 Acked 1025 Flush 0 Done Sent Done false
48342024-01-10T17:57:04.983Z 27 Acked 1026 Write 4 Done Sent Done false
48352024-01-10T17:57:04.983Z 28 Acked 1027 Write 2 Done Sent Done false
48362024-01-10T17:57:04.983Z 29 Acked 1028 Write 3 Done Sent Done false
48372024-01-10T17:57:04.983Z 30 Acked 1029 Write 9 Sent Sent Sent false
48382024-01-10T17:57:04.983Z STATES DS:0 DS:1 DS:2 TOTAL
48392024-01-10T17:57:04.983Z New 0 0 0 0
48402024-01-10T17:57:04.983Z Sent 1 29 1 31
48412024-01-10T17:57:04.983Z Done 29 1 29 59
48422024-01-10T17:57:04.983Z Skipped 0 0 0 0
48432024-01-10T17:57:04.983Z Error 0 0 0 0
48442024-01-10T17:57:04.983ZLast Flush: 1025 0 1025
48452024-01-10T17:57:04.983ZDownstairs last five completed:
48462024-01-10T17:57:04.983ZUpstairs last five completed: 30 29 28 27 26
48472024-01-10T17:57:04.983ZWrote out file "/tmp/test_repair_verify.data"
48482024-01-10T17:57:04.987Z
48492024-01-10T17:57:04.990ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48502024-01-10T17:57:05.014ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48512024-01-10T17:57:05.015Z 0 000-019 32 32 32 62 62 62 T F T <---
48522024-01-10T17:57:05.015Z 1 020-039 44 40 44 82 73 82 F F F <---
48532024-01-10T17:57:05.015Z 2 040-059 38 38 38 71 71 71 F F F
48542024-01-10T17:57:05.015Z 3 060-079 40 40 40 73 73 73 F F F
48552024-01-10T17:57:05.019Z 4 080-099 38 38 38 71 71 71 F F F
48562024-01-10T17:57:05.019Z 5 100-119 38 38 38 71 71 71 T F T <---
48572024-01-10T17:57:05.019Z 6 120-139 44 42 44 84 79 84 F F F <---
48582024-01-10T17:57:05.019Z 7 140-159 44 42 44 84 79 84 F F F <---
48592024-01-10T17:57:05.019Z 8 160-179 43 43 43 81 81 81 T F T <---
48602024-01-10T17:57:05.021Z 9 180-199 42 42 42 77 77 77 F F F
48612024-01-10T17:57:05.021Z 10 200-219 34 34 34 65 65 65 F F F
48622024-01-10T17:57:05.021Z 11 220-239 40 40 40 75 75 75 F F F
48632024-01-10T17:57:05.021Z 12 240-259 40 40 40 75 75 75 F F F
48642024-01-10T17:57:05.021Z 13 260-279 42 42 42 77 77 77 F F F
48652024-01-10T17:57:05.021Z 14 280-299 44 38 44 82 71 82 F F F <---
48662024-01-10T17:57:05.021Z 15 300-319 44 40 44 83 73 83 F F F <---
48672024-01-10T17:57:05.021Z 16 320-339 37 37 37 70 70 70 F F F
48682024-01-10T17:57:05.021Z 17 340-359 42 42 42 80 80 80 F F F
48692024-01-10T17:57:05.021Z 18 360-379 44 42 44 85 78 85 F F F <---
48702024-01-10T17:57:05.021Z 19 380-399 42 42 42 77 77 77 F F F
48712024-01-10T17:57:05.021Z 20 400-419 42 42 42 77 77 77 F F F
48722024-01-10T17:57:05.021Z 21 420-439 42 42 42 77 77 77 F F F
48732024-01-10T17:57:05.021Z 22 440-459 38 38 38 71 71 71 F F F
48742024-01-10T17:57:05.021Z 23 460-479 43 43 43 81 81 81 F F F
48752024-01-10T17:57:05.021Z 24 480-499 43 43 43 81 81 81 F F F
48762024-01-10T17:57:05.021Z 25 500-519 44 42 44 83 80 83 T F T <---
48772024-01-10T17:57:05.022Z 26 520-539 44 39 44 84 72 84 F F F <---
48782024-01-10T17:57:05.022Z 27 540-559 44 40 44 84 74 84 F T F <---
48792024-01-10T17:57:05.022Z 28 560-579 40 40 40 74 74 74 F F F
48802024-01-10T17:57:05.022Z 29 580-599 44 41 44 84 76 84 F F F <---
48812024-01-10T17:57:05.022ZMax gen: 44, Max flush: 85
48822024-01-10T17:57:05.022ZError: Difference in extent metadata found!
48832024-01-10T17:57:05.022ZA Difference in extent metadata is expected here
48842024-01-10T17:57:05.022ZOn loop 22
48852024-01-10T17:57:05.022Z
48862024-01-10T17:57:05.026ZVerifying data now
48872024-01-10T17:57:10.295ZNo change after verify
48882024-01-10T17:57:10.295ZLoop: 22 Downstairs dump after verify (and repair):
48892024-01-10T17:57:10.318ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48902024-01-10T17:57:10.318Z 0 000-019 45 45 45 86 86 86 F F F
48912024-01-10T17:57:10.321Z 1 020-039 44 44 44 82 82 82 F F F
48922024-01-10T17:57:10.321Z 2 040-059 38 38 38 71 71 71 F F F
48932024-01-10T17:57:10.321Z 3 060-079 40 40 40 73 73 73 F F F
48942024-01-10T17:57:10.321Z 4 080-099 38 38 38 71 71 71 F F F
48952024-01-10T17:57:10.321Z 5 100-119 45 45 45 86 86 86 F F F
48962024-01-10T17:57:10.321Z 6 120-139 44 44 44 84 84 84 F F F
48972024-01-10T17:57:10.321Z 7 140-159 44 44 44 84 84 84 F F F
48982024-01-10T17:57:10.321Z 8 160-179 45 45 45 86 86 86 F F F
48992024-01-10T17:57:10.321Z 9 180-199 42 42 42 77 77 77 F F F
49002024-01-10T17:57:10.321Z 10 200-219 34 34 34 65 65 65 F F F
49012024-01-10T17:57:10.321Z 11 220-239 40 40 40 75 75 75 F F F
49022024-01-10T17:57:10.321Z 12 240-259 40 40 40 75 75 75 F F F
49032024-01-10T17:57:10.321Z 13 260-279 42 42 42 77 77 77 F F F
49042024-01-10T17:57:10.322Z 14 280-299 44 44 44 82 82 82 F F F
49052024-01-10T17:57:10.322Z 15 300-319 44 44 44 83 83 83 F F F
49062024-01-10T17:57:10.322Z 16 320-339 37 37 37 70 70 70 F F F
49072024-01-10T17:57:10.322Z 17 340-359 42 42 42 80 80 80 F F F
49082024-01-10T17:57:10.322Z 18 360-379 44 44 44 85 85 85 F F F
49092024-01-10T17:57:10.322Z 19 380-399 42 42 42 77 77 77 F F F
49102024-01-10T17:57:10.322Z 20 400-419 42 42 42 77 77 77 F F F
49112024-01-10T17:57:10.322Z 21 420-439 42 42 42 77 77 77 F F F
49122024-01-10T17:57:10.322Z 22 440-459 38 38 38 71 71 71 F F F
49132024-01-10T17:57:10.322Z 23 460-479 43 43 43 81 81 81 F F F
49142024-01-10T17:57:10.322Z 24 480-499 43 43 43 81 81 81 F F F
49152024-01-10T17:57:10.322Z 25 500-519 45 45 45 86 86 86 F F F
49162024-01-10T17:57:10.322Z 26 520-539 44 44 44 84 84 84 F F F
49172024-01-10T17:57:10.322Z 27 540-559 44 44 44 84 84 84 F F F
49182024-01-10T17:57:10.322Z 28 560-579 40 40 40 74 74 74 F F F
49192024-01-10T17:57:10.322Z 29 580-599 44 44 44 84 84 84 F F F
49202024-01-10T17:57:10.322ZMax gen: 45, Max flush: 86
49212024-01-10T17:57:10.326Z
49222024-01-10T17:57:10.326ZBegin loop 23
49232024-01-10T17:57:10.326ZDownstairs to restart: 0
49242024-01-10T17:57:10.352Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.352018122Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339}
49252024-01-10T17:57:10.352Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.352554283Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339}
49262024-01-10T17:57:10.358Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.352623882Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339}
49272024-01-10T17:57:10.358Z{"msg":"Crucible 519c607f-aae4-402c-8cb4-96194d7f69ee has session id: 0e9a93a1-b632-4f08-aa3b-79120e965563","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.352694445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49282024-01-10T17:57:10.358Z{"msg":"Upstairs opts: Upstairs UUID: 519c607f-aae4-402c-8cb4-96194d7f69ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.352755534Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49292024-01-10T17:57:10.358Z{"msg":"Crucible stats registered with UUID: 519c607f-aae4-402c-8cb4-96194d7f69ee","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.352836223Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339Crucible runtime is spawned
49302024-01-10T17:57:10.358Z,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49312024-01-10T17:57:10.358Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353034544Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339}
49322024-01-10T17:57:10.358Z{"msg":"519c607f-aae4-402c-8cb4-96194d7f69ee active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353112326Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49332024-01-10T17:57:10.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353159878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"0","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49342024-01-10T17:57:10.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353223115Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"0","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49352024-01-10T17:57:10.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353262629Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"1","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49362024-01-10T17:57:10.359Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353286448Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"1","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49372024-01-10T17:57:10.359Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353357117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"2","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49382024-01-10T17:57:10.359Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353402566Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"2","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49392024-01-10T17:57:10.359Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353496602Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49402024-01-10T17:57:10.359Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353545207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49412024-01-10T17:57:10.359Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353587342Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49422024-01-10T17:57:10.359Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.353713316Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49432024-01-10T17:57:10.359Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.35394961Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49442024-01-10T17:57:10.360Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.354032181Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49452024-01-10T17:57:10.360Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.355227241Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"1","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49462024-01-10T17:57:10.360Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.355342485Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"1","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49472024-01-10T17:57:10.361Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.355390034Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"2","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49482024-01-10T17:57:10.361Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.35543364Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"2","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49492024-01-10T17:57:10.361Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.355777259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"0","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49502024-01-10T17:57:10.361Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.355852705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"0","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49512024-01-10T17:57:10.361Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.356459912Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"1","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49522024-01-10T17:57:10.361Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.356606827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"2","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49532024-01-10T17:57:10.362Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.357067389Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"0","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49542024-01-10T17:57:10.367Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.357501957Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"1","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49552024-01-10T17:57:10.367Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.357585763Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"1","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49562024-01-10T17:57:10.368Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.357630579Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49572024-01-10T17:57:10.368Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.357697159Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"2","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49582024-01-10T17:57:10.368Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.357740355Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"2","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49592024-01-10T17:57:10.368Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.357806733Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49602024-01-10T17:57:10.368Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.357856261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"0","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49612024-01-10T17:57:10.368Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.357928499Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"0","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49622024-01-10T17:57:10.368Z{"msg":"[0]R flush_numbers[0..12]: [86, 82, 71, 73, 71, 86, 84, 84, 86, 77, 65, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.357974694Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49632024-01-10T17:57:10.368Z{"msg":"[0]R generation[0..12]: [45, 44, 38, 40, 38, 45, 44, 44, 45, 42, 34, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358060115Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49642024-01-10T17:57:10.368Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358090944Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49652024-01-10T17:57:10.368Z{"msg":"[1]R flush_numbers[0..12]: [86, 82, 71, 73, 71, 86, 84, 84, 86, 77, 65, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358115618Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49662024-01-10T17:57:10.368Z{"msg":"[1]R generation[0..12]: [45, 44, 38, 40, 38, 45, 44, 44, 45, 42, 34, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358138823Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49672024-01-10T17:57:10.368Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358185507Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49682024-01-10T17:57:10.368Z{"msg":"[2]R flush_numbers[0..12]: [86, 82, 71, 73, 71, 86, 84, 84, 86, 77, 65, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358210079Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49692024-01-10T17:57:10.369ZWait for a query_work_queue command to finish before sending IO
49702024-01-10T17:57:10.369Z{"msg":"[2]R generation[0..12]: [45, 44, 38, 40, 38, 45, 44, 44, 45, 42, 34, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358259026Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49712024-01-10T17:57:10.369Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.35828418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49722024-01-10T17:57:10.369Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358323962Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49732024-01-10T17:57:10.369Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358347144Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49742024-01-10T17:57:10.369Z{"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358386074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49752024-01-10T17:57:10.369Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358408895Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"":"downstairs","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49762024-01-10T17:57:10.369Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358473938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49772024-01-10T17:57:10.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358499225Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"0","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49782024-01-10T17:57:10.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358520774Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"1","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49792024-01-10T17:57:10.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358544751Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"client":"2","":"downstairs","session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49802024-01-10T17:57:10.369Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358573401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49812024-01-10T17:57:10.369Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358594128Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49822024-01-10T17:57:10.369Z{"msg":"519c607f-aae4-402c-8cb4-96194d7f69ee is now active with session: 0e9a93a1-b632-4f08-aa3b-79120e965563","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358615193Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49832024-01-10T17:57:10.369Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358634474Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339,"session_id":"0e9a93a1-b632-4f08-aa3b-79120e965563"}
49842024-01-10T17:57:10.369Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:10.358654617Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1339}
49852024-01-10T17:57:10.369ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
49862024-01-10T17:57:10.369ZLoading write count information from file "/tmp/test_repair_verify.data"
49872024-01-10T17:57:10.369ZRun Repair workload
49882024-01-10T17:57:10.370Z01/30 Write block 12 len 12288 data: 5 5 7
49892024-01-10T17:57:10.370Z02/30 Read block 41 len 20480
49902024-01-10T17:57:10.370Z03/30 Write block 354 len 4096 data: 3
49912024-01-10T17:57:10.370Z04/30 Flush
49922024-01-10T17:57:10.374Z05/30 Write block 530 len 36864 data: 4 6 6 6 5 5 6 6 6
49932024-01-10T17:57:10.375Z06/30 Write block 274 len 40960 data: 5 6 6 5 4 4 4 4 4 5
49942024-01-10T17:57:10.375Z07/30 Write block 445 len 24576 data: 10 10 7 6 5 6
49952024-01-10T17:57:10.375Z08/30 Read block 222 len 40960
49962024-01-10T17:57:10.378Z09/30 Read block 437 len 12288
49972024-01-10T17:57:10.378Z10/30 Read block 577 len 4096
49982024-01-10T17:57:10.381Z11/30 Read block 256 len 16384
49992024-01-10T17:57:10.381Z12/30 Read block 115 len 40960
50002024-01-10T17:57:10.384Z13/30 Read block 191 len 8192
50012024-01-10T17:57:10.387Z14/30 Write block 404 len 16384 data: 4 4 5 6
50022024-01-10T17:57:10.387Z15/30 Write block 453 len 12288 data: 4 3 3
50032024-01-10T17:57:10.387Z16/30 Read block 427 len 32768
50042024-01-10T17:57:10.390Z17/30 Flush
50052024-01-10T17:57:10.393Z18/30 Write block 418 len 12288 data: 5 9 9
50062024-01-10T17:57:10.393Z19/30 Read block 501 len 4096
50072024-01-10T17:57:10.396Z20/30 Write block 194 len 20480 data: 6 6 3 4 5
50082024-01-10T17:57:10.399Z21/30 Write block 256 len 36864 data: 3 3 4 3 3 3 2 3 3
50092024-01-10T17:57:10.399Z22/30 Write block 184 len 36864 data: 6 6 6 5 6 6 6 5 5
50102024-01-10T17:57:10.402Z23/30 Write block 130 len 16384 data: 4 4 4 4
50112024-01-10T17:57:10.402Z24/30 Read block 381 len 20480
50122024-01-10T17:57:10.411Z25/30 Write block 210 len 32768 data: 4 5 5 5 5 4 5 4
50132024-01-10T17:57:10.411Z26/30 Write block 267 len 24576 data: 5 5 6 6 6 4
50142024-01-10T17:57:10.415Z27/30 Write block 549 len 20480 data: 8 7 6 9 8
50152024-01-10T17:57:10.415Z28/30 Write block 543 len 24576 data: 4 5 6 6 8 8
50162024-01-10T17:57:10.416Z29/30 Write block 454 len 4096 data: 4
50172024-01-10T17:57:10.416Z30/30 Write block 251 len 36864 data: 4 5 4 2 3 4 4 5 4
50182024-01-10T17:57:10.416Z----------------------------------------------------------------
50192024-01-10T17:57:10.416Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50202024-01-10T17:57:10.416ZStates: Active Active Active
50212024-01-10T17:57:10.416ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50222024-01-10T17:57:10.416Z 1 Acked 1000 Write 3 Sent Done Done false
50232024-01-10T17:57:10.416Z 2 Acked 1001 Read 5 Sent Done Done false
50242024-01-10T17:57:10.416Z 3 Acked 1002 Write 1 Sent Done Done false
50252024-01-10T17:57:10.416Z 4 Acked 1003 Flush 0 Sent Done Done false
50262024-01-10T17:57:10.416Z 5 Acked 1004 Write 9 Sent Done Done false
50272024-01-10T17:57:10.416Z 6 Acked 1005 Write 10 Sent Done Done false
50282024-01-10T17:57:10.416Z 7 Acked 1006 Write 6 Sent Done Done false
50292024-01-10T17:57:10.416Z 8 Acked 1007 Read 10 Sent Done Done false
50302024-01-10T17:57:10.416Z 9 Acked 1008 Read 3 Sent Done Done false
50312024-01-10T17:57:10.416Z 10 Acked 1009 Read 1 Sent Done Done false
50322024-01-10T17:57:10.416Z 11 Acked 1010 Read 4 Sent Done Done false
50332024-01-10T17:57:10.416Z 12 Acked 1011 Read 10 Sent Done Done false
50342024-01-10T17:57:10.416Z 13 Acked 1012 Read 2 Sent Done Done false
50352024-01-10T17:57:10.416Z 14 Acked 1013 Write 4 Sent Done Done false
50362024-01-10T17:57:10.416Z 15 Acked 1014 Write 3 Sent Done Done false
50372024-01-10T17:57:10.417Z 16 Acked 1015 Read 8 Sent Done Done false
50382024-01-10T17:57:10.417Z 17 Acked 1016 Flush 0 Sent Done Done false
50392024-01-10T17:57:10.417Z 18 Acked 1017 Write 3 Sent Done Done false
50402024-01-10T17:57:10.417Z 19 Acked 1018 Read 1 Sent Done Done false
50412024-01-10T17:57:10.417Z 20 Acked 1019 Write 5 Sent Done Done false
50422024-01-10T17:57:10.417Z 21 Acked 1020 Write 9 Sent Done Done false
50432024-01-10T17:57:10.418Z 22 Acked 1021 Write 9 Sent Done Done false
50442024-01-10T17:57:10.418Z 23 Acked 1022 Write 4 Sent Done Done false
50452024-01-10T17:57:10.418Z 24 Acked 1023 Read 5 Sent Done Done false
50462024-01-10T17:57:10.419Z 25 Acked 1024 Write 8 Sent Done Done false
50472024-01-10T17:57:10.419Z 26 Acked 1025 Write 6 Sent Done Done false
50482024-01-10T17:57:10.419Z 27 Acked 1026 Write 5 Sent Done Done false
50492024-01-10T17:57:10.419Z 28 Acked 1027 Write 6 Sent Done Done false
50502024-01-10T17:57:10.419Z 29 Acked 1028 Write 1 Sent Done Done false
50512024-01-10T17:57:10.419Z 30 Acked 1029 Write 9 Sent Sent Sent false
50522024-01-10T17:57:10.419Z STATES DS:0 DS:1 DS:2 TOTAL
50532024-01-10T17:57:10.419Z New 0 0 0 0
50542024-01-10T17:57:10.419Z Sent 30 1 1 32
50552024-01-10T17:57:10.419Z Done 0 29 29 58
50562024-01-10T17:57:10.419Z Skipped 0 0 0 0
50572024-01-10T17:57:10.419Z Error 0 0 0 0
50582024-01-10T17:57:10.419ZLast Flush: 0 1016 1016
50592024-01-10T17:57:10.419ZDownstairs last five completed:
50602024-01-10T17:57:10.419ZUpstairs last five completed: 30 29 28 27 26
50612024-01-10T17:57:10.428ZWrote out file "/tmp/test_repair_verify.data"
50622024-01-10T17:57:10.428Z
50632024-01-10T17:57:10.428ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
50642024-01-10T17:57:10.452ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
50652024-01-10T17:57:10.452Z 0 000-019 45 46 46 86 87 87 F F F <---
50662024-01-10T17:57:10.452Z 1 020-039 44 44 44 82 82 82 F F F
50672024-01-10T17:57:10.452Z 2 040-059 38 38 38 71 71 71 F F F
50682024-01-10T17:57:10.452Z 3 060-079 40 40 40 73 73 73 F F F
50692024-01-10T17:57:10.452Z 4 080-099 38 38 38 71 71 71 F F F
50702024-01-10T17:57:10.455Z 5 100-119 45 45 45 86 86 86 F F F
50712024-01-10T17:57:10.455Z 6 120-139 44 44 44 84 84 84 F T T <---
50722024-01-10T17:57:10.455Z 7 140-159 44 44 44 84 84 84 F F F
50732024-01-10T17:57:10.455Z 8 160-179 45 45 45 86 86 86 F F F
50742024-01-10T17:57:10.455Z 9 180-199 42 42 42 77 77 77 F T T <---
50752024-01-10T17:57:10.456Z 10 200-219 34 34 34 65 65 65 F T T <---
50762024-01-10T17:57:10.456Z 11 220-239 40 40 40 75 75 75 F F F
50772024-01-10T17:57:10.456Z 12 240-259 40 40 40 75 75 75 F T T <---
50782024-01-10T17:57:10.456Z 13 260-279 42 46 46 77 88 88 F T T <---
50792024-01-10T17:57:10.456Z 14 280-299 44 46 46 82 88 88 F F F <---
50802024-01-10T17:57:10.456Z 15 300-319 44 44 44 83 83 83 F F F
50812024-01-10T17:57:10.456Z 16 320-339 37 37 37 70 70 70 F F F
50822024-01-10T17:57:10.456Z 17 340-359 42 46 46 80 87 87 F F F <---
50832024-01-10T17:57:10.457Z 18 360-379 44 44 44 85 85 85 F F F
50842024-01-10T17:57:10.457Z 19 380-399 42 42 42 77 77 77 F F F
50852024-01-10T17:57:10.457Z 20 400-419 42 46 46 77 88 88 F T T <---
50862024-01-10T17:57:10.457Z 21 420-439 42 42 42 77 77 77 F T T <---
50872024-01-10T17:57:10.457Z 22 440-459 38 46 46 71 88 88 F T T <---
50882024-01-10T17:57:10.457Z 23 460-479 43 43 43 81 81 81 F F F
50892024-01-10T17:57:10.457Z 24 480-499 43 43 43 81 81 81 F F F
50902024-01-10T17:57:10.457Z 25 500-519 45 45 45 86 86 86 F F F
50912024-01-10T17:57:10.457Z 26 520-539 44 46 46 84 88 88 F F F <---
50922024-01-10T17:57:10.457Z 27 540-559 44 44 44 84 84 84 F T T <---
50932024-01-10T17:57:10.457Z 28 560-579 40 40 40 74 74 74 F F F
50942024-01-10T17:57:10.457Z 29 580-599 44 44 44 84 84 84 F F F
50952024-01-10T17:57:10.457ZMax gen: 46, Max flush: 88
50962024-01-10T17:57:10.457ZError: Difference in extent metadata found!
50972024-01-10T17:57:10.461ZA Difference in extent metadata is expected here
50982024-01-10T17:57:10.461ZOn loop 23
50992024-01-10T17:57:10.461Z
51002024-01-10T17:57:10.461ZVerifying data now
51012024-01-10T17:57:15.749ZNo change after verify
51022024-01-10T17:57:15.749ZLoop: 23 Downstairs dump after verify (and repair):
51032024-01-10T17:57:15.760ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51042024-01-10T17:57:15.761Z 0 000-019 46 46 46 87 87 87 F F F
51052024-01-10T17:57:15.761Z 1 020-039 44 44 44 82 82 82 F F F
51062024-01-10T17:57:15.761Z 2 040-059 38 38 38 71 71 71 F F F
51072024-01-10T17:57:15.761Z 3 060-079 40 40 40 73 73 73 F F F
51082024-01-10T17:57:15.762Z 4 080-099 38 38 38 71 71 71 F F F
51092024-01-10T17:57:15.762Z 5 100-119 45 45 45 86 86 86 F F F
51102024-01-10T17:57:15.773Z 6 120-139 47 47 47 89 89 89 F F F
51112024-01-10T17:57:15.773Z 7 140-159 44 44 44 84 84 84 F F F
51122024-01-10T17:57:15.774Z 8 160-179 45 45 45 86 86 86 F F F
51132024-01-10T17:57:15.774Z 9 180-199 47 47 47 89 89 89 F F F
51142024-01-10T17:57:15.774Z 10 200-219 47 47 47 89 89 89 F F F
51152024-01-10T17:57:15.774Z 11 220-239 40 40 40 75 75 75 F F F
51162024-01-10T17:57:15.774Z 12 240-259 47 47 47 89 89 89 F F F
51172024-01-10T17:57:15.774Z 13 260-279 47 47 47 89 89 89 F F F
51182024-01-10T17:57:15.774Z 14 280-299 46 46 46 88 88 88 F F F
51192024-01-10T17:57:15.774Z 15 300-319 44 44 44 83 83 83 F F F
51202024-01-10T17:57:15.774Z 16 320-339 37 37 37 70 70 70 F F F
51212024-01-10T17:57:15.774Z 17 340-359 46 46 46 87 87 87 F F F
51222024-01-10T17:57:15.774Z 18 360-379 44 44 44 85 85 85 F F F
51232024-01-10T17:57:15.774Z 19 380-399 42 42 42 77 77 77 F F F
51242024-01-10T17:57:15.774Z 20 400-419 47 47 47 89 89 89 F F F
51252024-01-10T17:57:15.775Z 21 420-439 47 47 47 89 89 89 F F F
51262024-01-10T17:57:15.775Z 22 440-459 47 47 47 89 89 89 F F F
51272024-01-10T17:57:15.775Z 23 460-479 43 43 43 81 81 81 F F F
51282024-01-10T17:57:15.775Z 24 480-499 43 43 43 81 81 81 F F F
51292024-01-10T17:57:15.775Z 25 500-519 45 45 45 86 86 86 F F F
51302024-01-10T17:57:15.775Z 26 520-539 46 46 46 88 88 88 F F F
51312024-01-10T17:57:15.775Z 27 540-559 47 47 47 89 89 89 F F F
51322024-01-10T17:57:15.775Z 28 560-579 40 40 40 74 74 74 F F F
51332024-01-10T17:57:15.775Z 29 580-599 44 44 44 84 84 84 F F F
51342024-01-10T17:57:15.775ZMax gen: 47, Max flush: 89
51352024-01-10T17:57:15.775Z
51362024-01-10T17:57:15.775ZBegin loop 24
51372024-01-10T17:57:15.776ZDownstairs to restart: 0
51382024-01-10T17:57:15.795Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.79359209Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347}
51392024-01-10T17:57:15.806Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.794099427Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347}
51402024-01-10T17:57:15.806Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.794156216Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347}
51412024-01-10T17:57:15.806Z{"msg":"Crucible 9b676123-9770-4d13-9d13-c300cf741e05 has session id: acddb347-c1ad-4e78-96d9-1e9c77e59876","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.794179382Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51422024-01-10T17:57:15.806Z{"msg":"Upstairs opts: Upstairs UUID: 9b676123-9770-4d13-9d13-c300cf741e05, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.794226641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51432024-01-10T17:57:15.806Z{"msg":"Crucible stats registered with UUID: 9b676123-9770-4d13-9d13-c300cf741e05","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.794295245Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51442024-01-10T17:57:15.806ZCrucible runtime is spawned
51452024-01-10T17:57:15.806Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.794644352Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347}
51462024-01-10T17:57:15.806Z{"msg":"9b676123-9770-4d13-9d13-c300cf741e05 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.794739956Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51472024-01-10T17:57:15.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.794867015Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"0","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51482024-01-10T17:57:15.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.795006721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"0","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51492024-01-10T17:57:15.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.795066931Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"1","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51502024-01-10T17:57:15.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.795108371Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"1","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51512024-01-10T17:57:15.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.79517832Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"2","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51522024-01-10T17:57:15.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.795225403Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"2","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51532024-01-10T17:57:15.807Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.795294283Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51542024-01-10T17:57:15.807Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.795370523Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51552024-01-10T17:57:15.807Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.795542207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51562024-01-10T17:57:15.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.795607611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51572024-01-10T17:57:15.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.795634865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51582024-01-10T17:57:15.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.795677236Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51592024-01-10T17:57:15.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.796648406Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"1","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51602024-01-10T17:57:15.807Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.796754517Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"1","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51612024-01-10T17:57:15.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.796806595Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"2","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51622024-01-10T17:57:15.808Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.796951787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"2","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51632024-01-10T17:57:15.808Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.797730603Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"2","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51642024-01-10T17:57:15.808Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.797972923Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"1","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51652024-01-10T17:57:15.808Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.798049491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"0","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51662024-01-10T17:57:15.808Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.798092512Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"0","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51672024-01-10T17:57:15.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.798436624Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"2","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51682024-01-10T17:57:15.808Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.798520471Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"2","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51692024-01-10T17:57:15.808Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.798565712Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51702024-01-10T17:57:15.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.798942631Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"1","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51712024-01-10T17:57:15.808Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.799018566Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"1","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51722024-01-10T17:57:15.809Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.799044333Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51732024-01-10T17:57:15.809Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.799504523Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"0","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51742024-01-10T17:57:15.809Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800218422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"0","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51752024-01-10T17:57:15.809Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800311808Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"0","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51762024-01-10T17:57:15.809Z{"msg":"[0]R flush_numbers[0..12]: [87, 82, 71, 73, 71, 86, 89, 84, 86, 89, 89, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800375994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51772024-01-10T17:57:15.809Z{"msg":"[0]R generation[0..12]: [46, 44, 38, 40, 38, 45, 47, 44, 45, 47, 47, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800403749Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51782024-01-10T17:57:15.809Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800446458Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51792024-01-10T17:57:15.809Z{"msg":"[1]R flush_numbers[0..12]: [87, 82, 71, 73, 71, 86, 89, 84, 86, 89, 89, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.80047074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51802024-01-10T17:57:15.809Z{"msg":"[1]R generation[0..12]: [46, 44, 38, 40, 38, 45, 47, 44, 45, 47, 47, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.80051122Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51812024-01-10T17:57:15.809Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800536094Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51822024-01-10T17:57:15.809Z{"msg":"[2]R flush_numbers[0..12]: [87, 82, 71, 73, 71, 86, 89, 84, 86, 89, 89, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800587516Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51832024-01-10T17:57:15.809Z{"msg":"[2]R generation[0..12]: [46, 44, 38, 40, 38, 45, 47, 44, 45, 47, 47, 40]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
51842024-01-10T17:57:15.809Z,"time":"2024-01-10T17:57:15.80061139Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51852024-01-10T17:57:15.810Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800639529Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51862024-01-10T17:57:15.810Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800679147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51872024-01-10T17:57:15.810Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800702311Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51882024-01-10T17:57:15.810Z{"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800746001Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51892024-01-10T17:57:15.810Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800768734Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"":"downstairs","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51902024-01-10T17:57:15.810Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800790336Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51912024-01-10T17:57:15.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.80080981Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"0","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51922024-01-10T17:57:15.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800831329Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"1","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51932024-01-10T17:57:15.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800870763Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"client":"2","":"downstairs","session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51942024-01-10T17:57:15.810Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800893388Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51952024-01-10T17:57:15.810Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800931133Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51962024-01-10T17:57:15.810Z{"msg":"9b676123-9770-4d13-9d13-c300cf741e05 is now active with session: acddb347-c1ad-4e78-96d9-1e9c77e59876","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800953392Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51972024-01-10T17:57:15.810Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.800990522Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347,"session_id":"acddb347-c1ad-4e78-96d9-1e9c77e59876"}
51982024-01-10T17:57:15.811Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:15.801011872Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1347}
51992024-01-10T17:57:15.812ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52002024-01-10T17:57:15.812ZLoading write count information from file "/tmp/test_repair_verify.data"
52012024-01-10T17:57:15.812ZRun Repair workload
52022024-01-10T17:57:15.829Z01/30 Write block 257 len 8192 data: 5 6
52032024-01-10T17:57:15.829Z02/30 Read block 354 len 24576
52042024-01-10T17:57:15.829Z03/30 Read block 467 len 16384
52052024-01-10T17:57:15.829Z04/30 Flush
52062024-01-10T17:57:15.829Z05/30 Write block 239 len 16384 data: 5 5 4 4
52072024-01-10T17:57:15.829Z06/30 Write block 0 len 20480 data: 3 3 4 5 5
52082024-01-10T17:57:15.829Z07/30 Read block 287 len 40960
52092024-01-10T17:57:15.835Z08/30 Write block 358 len 4096 data: 5
52102024-01-10T17:57:15.835Z09/30 Write block 199 len 28672 data: 6 5 5 5 4 3 3
52112024-01-10T17:57:15.835Z10/30 Read block 113 len 20480
52122024-01-10T17:57:15.835Z11/30 Write block 161 len 40960 data: 5 8 8 8 9 9 8 7 6 5
52132024-01-10T17:57:15.835Z12/30 Read block 524 len 20480
52142024-01-10T17:57:15.835Z13/30 Write block 244 len 12288 data: 6 5 3
52152024-01-10T17:57:15.835Z14/30 Read block 234 len 40960
52162024-01-10T17:57:15.835Z15/30 Read block 259 len 4096
52172024-01-10T17:57:15.835Z16/30 Read block 361 len 12288
52182024-01-10T17:57:15.835Z17/30 Write block 216 len 24576 data: 6 5 3 3 3 3
52192024-01-10T17:57:15.843Z18/30 Read block 505 len 20480
52202024-01-10T17:57:15.844Z19/30 Flush
52212024-01-10T17:57:15.844Z20/30 Write block 113 len 8192 data: 5 5
52222024-01-10T17:57:15.844Z21/30 Flush
52232024-01-10T17:57:15.851Z22/30 Write block 191 len 32768 data: 6 6 5 7 7 4 5 6
52242024-01-10T17:57:15.851Z23/30 Read block 11 len 36864
52252024-01-10T17:57:15.858Z24/30 Read block 168 len 8192
52262024-01-10T17:57:15.858Z25/30 Read block 18 len 24576
52272024-01-10T17:57:15.859Z26/30 Write block 351 len 8192 data: 4 4
52282024-01-10T17:57:15.859Z27/30 Read block 24 len 16384
52292024-01-10T17:57:15.859Z28/30 Write block 460 len 16384 data: 2 2 3 4
52302024-01-10T17:57:15.859Z29/30 Read block 254 len 20480
52312024-01-10T17:57:15.870Z30/30 Read block 584 len 16384
52322024-01-10T17:57:15.871Z----------------------------------------------------------------
52332024-01-10T17:57:15.871Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
52342024-01-10T17:57:15.872ZStates: Active Active Active
52352024-01-10T17:57:15.872ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52362024-01-10T17:57:15.872Z 1 Acked 1000 Write 2 Sent Done Done false
52372024-01-10T17:57:15.872Z 2 Acked 1001 Read 6 Sent Done Done false
52382024-01-10T17:57:15.872Z 3 Acked 1002 Read 4 Sent Done Done false
52392024-01-10T17:57:15.872Z 4 Acked 1003 Flush 0 Sent Done Done false
52402024-01-10T17:57:15.873Z 5 Acked 1004 Write 4 Sent Done Done false
52412024-01-10T17:57:15.873Z 6 Acked 1005 Write 5 Sent Done Done false
52422024-01-10T17:57:15.873Z 7 Acked 1006 Read 10 Sent Done Done false
52432024-01-10T17:57:15.873Z 8 Acked 1007 Write 1 Sent Done Done false
52442024-01-10T17:57:15.873Z 9 Acked 1008 Write 7 Sent Done Done false
52452024-01-10T17:57:15.874Z 10 Acked 1009 Read 5 Sent Done Done false
52462024-01-10T17:57:15.874Z 11 Acked 1010 Write 10 Sent Done Done false
52472024-01-10T17:57:15.874Z 12 Acked 1011 Read 5 Sent Done Done false
52482024-01-10T17:57:15.876Z 13 Acked 1012 Write 3 Sent Done Done false
52492024-01-10T17:57:15.876Z 14 Acked 1013 Read 10 Sent Done Done false
52502024-01-10T17:57:15.876Z 15 Acked 1014 Read 1 Sent Done Done false
52512024-01-10T17:57:15.877Z 16 Acked 1015 Read 3 Sent Done Done false
52522024-01-10T17:57:15.877Z 17 Acked 1016 Write 6 Sent Done Done false
52532024-01-10T17:57:15.877Z 18 Acked 1017 Read 5 Sent Done Done false
52542024-01-10T17:57:15.877Z 19 Acked 1018 Flush 0 Sent Done Done false
52552024-01-10T17:57:15.877Z 20 Acked 1019 Write 2 Sent Done Done false
52562024-01-10T17:57:15.877Z 21 Acked 1020 Flush 0 Sent Done Done false
52572024-01-10T17:57:15.878Z 22 Acked 1021 Write 8 Sent Done Done false
52582024-01-10T17:57:15.878Z 23 Acked 1022 Read 9 Sent Done Done false
52592024-01-10T17:57:15.878Z 24 Acked 1023 Read 2 Sent Done Done false
52602024-01-10T17:57:15.878Z 25 Acked 1024 Read 6 Sent Done Done false
52612024-01-10T17:57:15.878Z 26 Acked 1025 Write 2 Sent Done Done false
52622024-01-10T17:57:15.878Z 27 Acked 1026 Read 4 Sent Done Done false
52632024-01-10T17:57:15.878Z 28 Acked 1027 Write 4 Sent Done Done false
52642024-01-10T17:57:15.878Z 29 Acked 1028 Read 5 Sent Done Done false
52652024-01-10T17:57:15.878Z 30 Acked 1029 Read 4 Sent Sent Done false
52662024-01-10T17:57:15.879Z STATES DS:0 DS:1 DS:2 TOTAL
52672024-01-10T17:57:15.879Z New 0 0 0 0
52682024-01-10T17:57:15.879Z Sent 30 1 0 31
52692024-01-10T17:57:15.879Z Done 0 29 30 59
52702024-01-10T17:57:15.879Z Skipped 0 0 0 0
52712024-01-10T17:57:15.879Z Error 0 0 0 0
52722024-01-10T17:57:15.880ZLast Flush: 0 1020 1020
52732024-01-10T17:57:15.880ZDownstairs last five completed:
52742024-01-10T17:57:15.880ZUpstairs last five completed: 30 29 28 27 26
52752024-01-10T17:57:15.880ZWrote out file "/tmp/test_repair_verify.data"
52762024-01-10T17:57:15.890Z
52772024-01-10T17:57:15.890ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
52782024-01-10T17:57:15.896ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
52792024-01-10T17:57:15.896Z 0 000-019 46 48 48 87 91 91 F F F <---
52802024-01-10T17:57:15.896Z 1 020-039 44 44 44 82 82 82 F F F
52812024-01-10T17:57:15.896Z 2 040-059 38 38 38 71 71 71 F F F
52822024-01-10T17:57:15.896Z 3 060-079 40 40 40 73 73 73 F F F
52832024-01-10T17:57:15.900Z 4 080-099 38 38 38 71 71 71 F F F
52842024-01-10T17:57:15.900Z 5 100-119 45 48 48 86 92 92 F F F <---
52852024-01-10T17:57:15.900Z 6 120-139 47 47 47 89 89 89 F F F
52862024-01-10T17:57:15.900Z 7 140-159 44 44 44 84 84 84 F F F
52872024-01-10T17:57:15.900Z 8 160-179 45 48 48 86 91 91 F F F <---
52882024-01-10T17:57:15.900Z 9 180-199 47 48 48 89 91 91 F T T <---
52892024-01-10T17:57:15.900Z 10 200-219 47 48 48 89 91 91 F F F <---
52902024-01-10T17:57:15.900Z 11 220-239 40 48 48 75 91 91 F F F <---
52912024-01-10T17:57:15.900Z 12 240-259 47 48 48 89 91 91 F F F <---
52922024-01-10T17:57:15.900Z 13 260-279 47 47 47 89 89 89 F F F
52932024-01-10T17:57:15.900Z 14 280-299 46 46 46 88 88 88 F F F
52942024-01-10T17:57:15.900Z 15 300-319 44 44 44 83 83 83 F F F
52952024-01-10T17:57:15.901Z 16 320-339 37 37 37 70 70 70 F F F
52962024-01-10T17:57:15.901Z 17 340-359 46 48 48 87 91 91 F T T <---
52972024-01-10T17:57:15.902Z 18 360-379 44 44 44 85 85 85 F F F
52982024-01-10T17:57:15.902Z 19 380-399 42 42 42 77 77 77 F F F
52992024-01-10T17:57:15.902Z 20 400-419 47 47 47 89 89 89 F F F
53002024-01-10T17:57:15.902Z 21 420-439 47 47 47 89 89 89 F F F
53012024-01-10T17:57:15.902Z 22 440-459 47 47 47 89 89 89 F F F
53022024-01-10T17:57:15.902Z 23 460-479 43 43 43 81 81 81 F T T <---
53032024-01-10T17:57:15.902Z 24 480-499 43 43 43 81 81 81 F F F
53042024-01-10T17:57:15.902Z 25 500-519 45 45 45 86 86 86 F F F
53052024-01-10T17:57:15.902Z 26 520-539 46 46 46 88 88 88 F F F
53062024-01-10T17:57:15.902Z 27 540-559 47 47 47 89 89 89 F F F
53072024-01-10T17:57:15.902Z 28 560-579 40 40 40 74 74 74 F F F
53082024-01-10T17:57:15.902Z 29 580-599 44 44 44 84 84 84 F F F
53092024-01-10T17:57:15.903ZMax gen: 48, Max flush: 92
53102024-01-10T17:57:15.903ZError: Difference in extent metadata found!
53112024-01-10T17:57:15.906ZA Difference in extent metadata is expected here
53122024-01-10T17:57:15.906ZOn loop 24
53132024-01-10T17:57:15.906Z
53142024-01-10T17:57:15.906ZVerifying data now
53152024-01-10T17:57:20.863ZNo change after verify
53162024-01-10T17:57:20.863ZLoop: 24 Downstairs dump after verify (and repair):
53172024-01-10T17:57:20.886ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53182024-01-10T17:57:20.886Z 0 000-019 48 48 48 91 91 91 F F F
53192024-01-10T17:57:20.886Z 1 020-039 44 44 44 82 82 82 F F F
53202024-01-10T17:57:20.886Z 2 040-059 38 38 38 71 71 71 F F F
53212024-01-10T17:57:20.886Z 3 060-079 40 40 40 73 73 73 F F F
53222024-01-10T17:57:20.886Z 4 080-099 38 38 38 71 71 71 F F F
53232024-01-10T17:57:20.886Z 5 100-119 48 48 48 92 92 92 F F F
53242024-01-10T17:57:20.886Z 6 120-139 47 47 47 89 89 89 F F F
53252024-01-10T17:57:20.900Z 7 140-159 44 44 44 84 84 84 F F F
53262024-01-10T17:57:20.900Z 8 160-179 48 48 48 91 91 91 F F F
53272024-01-10T17:57:20.900Z 9 180-199 49 49 49 93 93 93 F F F
53282024-01-10T17:57:20.900Z 10 200-219 48 48 48 91 91 91 F F F
53292024-01-10T17:57:20.900Z 11 220-239 48 48 48 91 91 91 F F F
53302024-01-10T17:57:20.900Z 12 240-259 48 48 48 91 91 91 F F F
53312024-01-10T17:57:20.900Z 13 260-279 47 47 47 89 89 89 F F F
53322024-01-10T17:57:20.900Z 14 280-299 46 46 46 88 88 88 F F F
53332024-01-10T17:57:20.900Z 15 300-319 44 44 44 83 83 83 F F F
53342024-01-10T17:57:20.900Z 16 320-339 37 37 37 70 70 70 F F F
53352024-01-10T17:57:20.900Z 17 340-359 49 49 49 93 93 93 F F F
53362024-01-10T17:57:20.900Z 18 360-379 44 44 44 85 85 85 F F F
53372024-01-10T17:57:20.900Z 19 380-399 42 42 42 77 77 77 F F F
53382024-01-10T17:57:20.900Z 20 400-419 47 47 47 89 89 89 F F F
53392024-01-10T17:57:20.900Z 21 420-439 47 47 47 89 89 89 F F F
53402024-01-10T17:57:20.901Z 22 440-459 47 47 47 89 89 89 F F F
53412024-01-10T17:57:20.901Z 23 460-479 49 49 49 93 93 93 F F F
53422024-01-10T17:57:20.901Z 24 480-499 43 43 43 81 81 81 F F F
53432024-01-10T17:57:20.901Z 25 500-519 45 45 45 86 86 86 F F F
53442024-01-10T17:57:20.901Z 26 520-539 46 46 46 88 88 88 F F F
53452024-01-10T17:57:20.901Z 27 540-559 47 47 47 89 89 89 F F F
53462024-01-10T17:57:20.901Z 28 560-579 40 40 40 74 74 74 F F F
53472024-01-10T17:57:20.902Z 29 580-599 44 44 44 84 84 84 F F F
53482024-01-10T17:57:20.902ZMax gen: 49, Max flush: 93
53492024-01-10T17:57:20.902Z
53502024-01-10T17:57:20.902ZBegin loop 25
53512024-01-10T17:57:20.902ZDownstairs to restart: 1
53522024-01-10T17:57:20.921Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.921647325Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355}
53532024-01-10T17:57:20.921Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922176503Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355}
53542024-01-10T17:57:20.921Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922205952Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355}
53552024-01-10T17:57:20.926Z{"msg":"Crucible runtime is spawned
53562024-01-10T17:57:20.926ZCrucible f55193e8-67e5-4d05-b90c-625f18eaaf04 has session id: f376e36c-6f7c-40c4-9cb4-b29b75c0c220","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922373683Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53572024-01-10T17:57:20.926Z{"msg":"Upstairs opts: Upstairs UUID: f55193e8-67e5-4d05-b90c-625f18eaaf04, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922447802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53582024-01-10T17:57:20.926Z{"msg":"Crucible stats registered with UUID: f55193e8-67e5-4d05-b90c-625f18eaaf04","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922529208Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53592024-01-10T17:57:20.926Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922569502Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355}
53602024-01-10T17:57:20.926Z{"msg":"f55193e8-67e5-4d05-b90c-625f18eaaf04 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922617241Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53612024-01-10T17:57:20.926Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922640719Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"0","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53622024-01-10T17:57:20.927Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922689693Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"0","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53632024-01-10T17:57:20.927Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922713476Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"1","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53642024-01-10T17:57:20.927Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922774891Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"1","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53652024-01-10T17:57:20.927Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922811995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"2","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53662024-01-10T17:57:20.927Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922835675Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"2","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53672024-01-10T17:57:20.927Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922861925Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53682024-01-10T17:57:20.927Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.922942078Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53692024-01-10T17:57:20.927Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.923109474Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53702024-01-10T17:57:20.927Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.923310292Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53712024-01-10T17:57:20.927Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.923417282Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53722024-01-10T17:57:20.927Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.923571889Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53732024-01-10T17:57:20.927Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.924856537Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"0","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53742024-01-10T17:57:20.927Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.924914011Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"0","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53752024-01-10T17:57:20.927Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.92496166Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"2","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53762024-01-10T17:57:20.927Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.925005457Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"2","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53772024-01-10T17:57:20.933Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.925670864Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"1","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53782024-01-10T17:57:20.935Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.925736051Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"1","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53792024-01-10T17:57:20.935Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.925763975Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"0","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53802024-01-10T17:57:20.935Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.92595455Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"2","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53812024-01-10T17:57:20.935Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.926769603Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"0","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53822024-01-10T17:57:20.935Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.926862113Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"0","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53832024-01-10T17:57:20.935Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.92694737Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53842024-01-10T17:57:20.935Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.927024497Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"2","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53852024-01-10T17:57:20.935Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.927104616Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"2","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53862024-01-10T17:57:20.935Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.927183069Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53872024-01-10T17:57:20.935Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.927222641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"1","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53882024-01-10T17:57:20.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.927846706Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"1","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53892024-01-10T17:57:20.936Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.927927469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"1","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53902024-01-10T17:57:20.936Z{"msg":"[0]R flush_numbers[0..12]: [91, 82, 71, 73, 71, 92, 89, 84, 91, 93, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.927965Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53912024-01-10T17:57:20.936Z{"msg":"[0]R generation[0..12]: [48, 44, 38, 40, 38, 48, 47, 44, 48, 49, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.927998203Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53922024-01-10T17:57:20.936Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.92802355Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53932024-01-10T17:57:20.936Z{"msg":"[1]R flush_numbers[0..12]: [91, 82, 71, 73, 71, 92, 89, 84, 91, 93, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928068049Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53942024-01-10T17:57:20.936Z{"msg":"[1]R generation[0..12]: [48, 44, 38, 40, 38, 48, 47, 44, 48, 49, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928092915Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53952024-01-10T17:57:20.936Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928134479Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53962024-01-10T17:57:20.936Z{"msg":"[2]R flush_numbers[0..12]: [91, 82, 71, 73, 71, 92, 89, 84, 91, 93, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928158855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53972024-01-10T17:57:20.938Z{"msg":"[2]R generation[0..12]: [48, 44, 38, 40, 38, 48, 47, 44, 48, 49, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928213293Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53982024-01-10T17:57:20.938Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928238073Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
53992024-01-10T17:57:20.938Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
54002024-01-10T17:57:20.938Z30,"time":"2024-01-10T17:57:20.928263079Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54012024-01-10T17:57:20.938Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928322723Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54022024-01-10T17:57:20.938Z{"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.92834699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54032024-01-10T17:57:20.938Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928368645Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"":"downstairs","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54042024-01-10T17:57:20.938Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928409353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54052024-01-10T17:57:20.938Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928430614Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"0","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54062024-01-10T17:57:20.938Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928452619Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"1","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54072024-01-10T17:57:20.938Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928492356Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"client":"2","":"downstairs","session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54082024-01-10T17:57:20.944Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928515247Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54092024-01-10T17:57:20.945Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928535232Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54102024-01-10T17:57:20.945Z{"msg":"f55193e8-67e5-4d05-b90c-625f18eaaf04 is now active with session: f376e36c-6f7c-40c4-9cb4-b29b75c0c220","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928574259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54112024-01-10T17:57:20.945Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928594734Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355,"session_id":"f376e36c-6f7c-40c4-9cb4-b29b75c0c220"}
54122024-01-10T17:57:20.945Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:20.928614799Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1355}
54132024-01-10T17:57:20.945ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54142024-01-10T17:57:20.945ZLoading write count information from file "/tmp/test_repair_verify.data"
54152024-01-10T17:57:20.945ZRun Repair workload
54162024-01-10T17:57:20.945Z01/30 Write block 256 len 16384 data: 5 6 7 5
54172024-01-10T17:57:20.948Z02/30 Read block 396 len 32768
54182024-01-10T17:57:20.949Z03/30 Write block 356 len 12288 data: 3 4 6
54192024-01-10T17:57:20.949Z04/30 Write block 117 len 16384 data: 4 4 5 6
54202024-01-10T17:57:20.949Z05/30 Write block 209 len 36864 data: 4 5 6 6 6 6 5 7 6
54212024-01-10T17:57:20.950Z06/30 Flush
54222024-01-10T17:57:20.950Z07/30 Write block 580 len 8192 data: 5 5
54232024-01-10T17:57:20.950Z08/30 Write block 222 len 32768 data: 3 2 2 3 3 3 3 3
54242024-01-10T17:57:20.950Z09/30 Write block 30 len 36864 data: 8 6 6 5 3 4 4 3 4
54252024-01-10T17:57:20.950Z10/30 Write block 560 len 20480 data: 3 4 5 6 6
54262024-01-10T17:57:20.950Z11/30 Write block 258 len 12288 data: 8 6 4
54272024-01-10T17:57:20.950Z12/30 Write block 445 len 12288 data: 11 11 8
54282024-01-10T17:57:20.953Z13/30 Read block 384 len 36864
54292024-01-10T17:57:20.953Z14/30 Write block 582 len 20480 data: 6 6 4 3 5
54302024-01-10T17:57:20.953Z15/30 Read block 110 len 28672
54312024-01-10T17:57:20.956Z16/30 Read block 339 len 16384
54322024-01-10T17:57:20.959Z17/30 Write block 529 len 12288 data: 4 5 7
54332024-01-10T17:57:20.959Z18/30 Read block 20 len 8192
54342024-01-10T17:57:20.962Z19/30 Flush
54352024-01-10T17:57:20.965Z20/30 Write block 216 len 32768 data: 8 7 4 4 4 4 4 3
54362024-01-10T17:57:20.968Z21/30 Write block 253 len 12288 data: 5 3 4
54372024-01-10T17:57:20.968Z22/30 Write block 160 len 20480 data: 4 6 9 9 9
54382024-01-10T17:57:20.972Z23/30 Write block 480 len 32768 data: 3 3 3 2 5 4 5 5
54392024-01-10T17:57:20.972Z24/30 Read block 215 len 32768
54402024-01-10T17:57:20.979Z25/30 Read block 109 len 32768
54412024-01-10T17:57:20.979Z26/30 Write block 305 len 12288 data: 4 4 3
54422024-01-10T17:57:20.979Z27/30 Read block 486 len 8192
54432024-01-10T17:57:20.995Z28/30 Read block 355 len 16384
54442024-01-10T17:57:20.995Z29/30 Read block 48 len 28672
54452024-01-10T17:57:20.995Z30/30 Read block 380 len 32768
54462024-01-10T17:57:20.995Z----------------------------------------------------------------
54472024-01-10T17:57:20.995Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
54482024-01-10T17:57:20.996ZStates: Active Active Active
54492024-01-10T17:57:20.996ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54502024-01-10T17:57:20.996Z 1 Acked 1000 Write 4 Done Done Done false
54512024-01-10T17:57:20.997Z 2 Acked 1001 Read 8 Done Sent Done false
54522024-01-10T17:57:20.997Z 3 Acked 1002 Write 3 Done Sent Done false
54532024-01-10T17:57:20.997Z 4 Acked 1003 Write 4 Done Sent Done false
54542024-01-10T17:57:20.997Z 5 Acked 1004 Write 9 Done Sent Done false
54552024-01-10T17:57:20.997Z 6 Acked 1005 Flush 0 Done Sent Done false
54562024-01-10T17:57:20.997Z 7 Acked 1006 Write 2 Done Sent Done false
54572024-01-10T17:57:20.997Z 8 Acked 1007 Write 8 Done Sent Done false
54582024-01-10T17:57:20.997Z 9 Acked 1008 Write 9 Done Sent Done false
54592024-01-10T17:57:20.998Z 10 Acked 1009 Write 5 Done Sent Done false
54602024-01-10T17:57:20.998Z 11 Acked 1010 Write 3 Done Sent Done false
54612024-01-10T17:57:20.998Z 12 Acked 1011 Write 3 Done Sent Done false
54622024-01-10T17:57:20.998Z 13 Acked 1012 Read 9 Done Sent Done false
54632024-01-10T17:57:20.998Z 14 Acked 1013 Write 5 Done Sent Done false
54642024-01-10T17:57:20.999Z 15 Acked 1014 Read 7 Done Sent Done false
54652024-01-10T17:57:20.999Z 16 Acked 1015 Read 4 Done Sent Done false
54662024-01-10T17:57:20.999Z 17 Acked 1016 Write 3 Done Sent Done false
54672024-01-10T17:57:21.000Z 18 Acked 1017 Read 2 Done Sent Done false
54682024-01-10T17:57:21.000Z 19 Acked 1018 Flush 0 Done Sent Done false
54692024-01-10T17:57:21.000Z 20 Acked 1019 Write 8 Done Sent Done false
54702024-01-10T17:57:21.000Z 21 Acked 1020 Write 3 Done Sent Done false
54712024-01-10T17:57:21.000Z 22 Acked 1021 Write 5 Done Sent Done false
54722024-01-10T17:57:21.000Z 23 Acked 1022 Write 8 Done Sent Done false
54732024-01-10T17:57:21.001Z 24 Acked 1023 Read 8 Done Sent Done false
54742024-01-10T17:57:21.001Z 25 Acked 1024 Read 8 Done Sent Done false
54752024-01-10T17:57:21.001Z 26 Acked 1025 Write 3 Done Sent Done false
54762024-01-10T17:57:21.001Z 27 Acked 1026 Read 2 Done Sent Done false
54772024-01-10T17:57:21.001Z 28 Acked 1027 Read 4 Done Sent Done false
54782024-01-10T17:57:21.001Z 29 Acked 1028 Read 7 Done Sent Done false
54792024-01-10T17:57:21.001Z 30 Acked 1029 Read 8 Sent Sent Done false
54802024-01-10T17:57:21.002Z STATES DS:0 DS:1 DS:2 TOTAL
54812024-01-10T17:57:21.002Z New 0 0 0 0
54822024-01-10T17:57:21.002Z Sent 1 29 0 30
54832024-01-10T17:57:21.002Z Done 29 1 30 60
54842024-01-10T17:57:21.002Z Skipped 0 0 0 0
54852024-01-10T17:57:21.002Z Error 0 0 0 0
54862024-01-10T17:57:21.002ZLast Flush: 1018 0 1018
54872024-01-10T17:57:21.002ZDownstairs last five completed:
54882024-01-10T17:57:21.002ZUpstairs last five completed: 30 29 28 27 26
54892024-01-10T17:57:21.002ZWrote out file "/tmp/test_repair_verify.data"
54902024-01-10T17:57:21.018Z
54912024-01-10T17:57:21.018ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
54922024-01-10T17:57:21.025ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
54932024-01-10T17:57:21.025Z 0 000-019 48 48 48 91 91 91 F F F
54942024-01-10T17:57:21.025Z 1 020-039 50 44 50 95 82 95 F F F <---
54952024-01-10T17:57:21.025Z 2 040-059 38 38 38 71 71 71 F F F
54962024-01-10T17:57:21.025Z 3 060-079 40 40 40 73 73 73 F F F
54972024-01-10T17:57:21.025Z 4 080-099 38 38 38 71 71 71 F F F
54982024-01-10T17:57:21.026Z 5 100-119 50 48 50 94 92 94 F F F <---
54992024-01-10T17:57:21.027Z 6 120-139 50 47 50 94 89 94 F F F <---
55002024-01-10T17:57:21.027Z 7 140-159 44 44 44 84 84 84 F F F
55012024-01-10T17:57:21.031Z 8 160-179 48 48 48 91 91 91 T F T <---
55022024-01-10T17:57:21.031Z 9 180-199 49 49 49 93 93 93 F F F
55032024-01-10T17:57:21.031Z 10 200-219 50 48 50 94 91 94 T F T <---
55042024-01-10T17:57:21.031Z 11 220-239 50 48 50 95 91 95 T F T <---
55052024-01-10T17:57:21.031Z 12 240-259 50 48 50 95 91 95 T T T <---
55062024-01-10T17:57:21.031Z 13 260-279 50 47 50 95 89 95 F F F <---
55072024-01-10T17:57:21.031Z 14 280-299 46 46 46 88 88 88 F F F
55082024-01-10T17:57:21.031Z 15 300-319 44 44 44 83 83 83 T F T <---
55092024-01-10T17:57:21.031Z 16 320-339 37 37 37 70 70 70 F F F
55102024-01-10T17:57:21.031Z 17 340-359 50 49 50 94 93 94 F F F <---
55112024-01-10T17:57:21.032Z 18 360-379 44 44 44 85 85 85 F F F
55122024-01-10T17:57:21.032Z 19 380-399 42 42 42 77 77 77 F F F
55132024-01-10T17:57:21.032Z 20 400-419 47 47 47 89 89 89 F F F
55142024-01-10T17:57:21.032Z 21 420-439 47 47 47 89 89 89 F F F
55152024-01-10T17:57:21.032Z 22 440-459 50 47 50 95 89 95 F F F <---
55162024-01-10T17:57:21.032Z 23 460-479 49 49 49 93 93 93 F F F
55172024-01-10T17:57:21.032Z 24 480-499 43 43 43 81 81 81 T F T <---
55182024-01-10T17:57:21.032Z 25 500-519 45 45 45 86 86 86 F F F
55192024-01-10T17:57:21.032Z 26 520-539 50 46 50 95 88 95 F F F <---
55202024-01-10T17:57:21.032Z 27 540-559 47 47 47 89 89 89 F F F
55212024-01-10T17:57:21.032Z 28 560-579 50 40 50 95 74 95 F F F <---
55222024-01-10T17:57:21.032Z 29 580-599 50 44 50 95 84 95 F F F <---
55232024-01-10T17:57:21.032ZMax gen: 50, Max flush: 95
55242024-01-10T17:57:21.032ZError: Difference in extent metadata found!
55252024-01-10T17:57:21.032ZA Difference in extent metadata is expected here
55262024-01-10T17:57:21.032ZOn loop 25
55272024-01-10T17:57:21.032Z
55282024-01-10T17:57:21.037ZVerifying data now
55292024-01-10T17:57:26.478ZNo change after verify
55302024-01-10T17:57:26.478ZLoop: 25 Downstairs dump after verify (and repair):
55312024-01-10T17:57:26.500ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55322024-01-10T17:57:26.500Z 0 000-019 48 48 48 91 91 91 F F F
55332024-01-10T17:57:26.500Z 1 020-039 50 50 50 95 95 95 F F F
55342024-01-10T17:57:26.502Z 2 040-059 38 38 38 71 71 71 F F F
55352024-01-10T17:57:26.502Z 3 060-079 40 40 40 73 73 73 F F F
55362024-01-10T17:57:26.502Z 4 080-099 38 38 38 71 71 71 F F F
55372024-01-10T17:57:26.502Z 5 100-119 50 50 50 94 94 94 F F F
55382024-01-10T17:57:26.502Z 6 120-139 50 50 50 94 94 94 F F F
55392024-01-10T17:57:26.502Z 7 140-159 44 44 44 84 84 84 F F F
55402024-01-10T17:57:26.502Z 8 160-179 51 51 51 96 96 96 F F F
55412024-01-10T17:57:26.502Z 9 180-199 49 49 49 93 93 93 F F F
55422024-01-10T17:57:26.502Z 10 200-219 51 51 51 96 96 96 F F F
55432024-01-10T17:57:26.502Z 11 220-239 51 51 51 96 96 96 F F F
55442024-01-10T17:57:26.502Z 12 240-259 51 51 51 96 96 96 F F F
55452024-01-10T17:57:26.502Z 13 260-279 50 50 50 95 95 95 F F F
55462024-01-10T17:57:26.502Z 14 280-299 46 46 46 88 88 88 F F F
55472024-01-10T17:57:26.502Z 15 300-319 51 51 51 96 96 96 F F F
55482024-01-10T17:57:26.502Z 16 320-339 37 37 37 70 70 70 F F F
55492024-01-10T17:57:26.502Z 17 340-359 50 50 50 94 94 94 F F F
55502024-01-10T17:57:26.506Z 18 360-379 44 44 44 85 85 85 F F F
55512024-01-10T17:57:26.506Z 19 380-399 42 42 42 77 77 77 F F F
55522024-01-10T17:57:26.506Z 20 400-419 47 47 47 89 89 89 F F F
55532024-01-10T17:57:26.506Z 21 420-439 47 47 47 89 89 89 F F F
55542024-01-10T17:57:26.506Z 22 440-459 50 50 50 95 95 95 F F F
55552024-01-10T17:57:26.506Z 23 460-479 49 49 49 93 93 93 F F F
55562024-01-10T17:57:26.506Z 24 480-499 51 51 51 96 96 96 F F F
55572024-01-10T17:57:26.506Z 25 500-519 45 45 45 86 86 86 F F F
55582024-01-10T17:57:26.506Z 26 520-539 50 50 50 95 95 95 F F F
55592024-01-10T17:57:26.508Z 27 540-559 47 47 47 89 89 89 F F F
55602024-01-10T17:57:26.508Z 28 560-579 50 50 50 95 95 95 F F F
55612024-01-10T17:57:26.508Z 29 580-599 50 50 50 95 95 95 F F F
55622024-01-10T17:57:26.508ZMax gen: 51, Max flush: 96
55632024-01-10T17:57:26.508Z
55642024-01-10T17:57:26.508ZBegin loop 26
55652024-01-10T17:57:26.508ZDownstairs to restart: 0
55662024-01-10T17:57:26.541Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.533896867Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363}
55672024-01-10T17:57:26.548Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.534453105Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363}
55682024-01-10T17:57:26.548Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.534519154Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363}
55692024-01-10T17:57:26.548Z{"msg":"Crucible runtime is spawned
55702024-01-10T17:57:26.548ZCrucible f379de3e-653e-446a-900f-3594124247bd has session id: c58b81b0-2b8c-4001-a2c0-5a975c6395fa","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.534685261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55712024-01-10T17:57:26.548Z{"msg":"Upstairs opts: Upstairs UUID: f379de3e-653e-446a-900f-3594124247bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.534804094Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55722024-01-10T17:57:26.548Z{"msg":"Crucible stats registered with UUID: f379de3e-653e-446a-900f-3594124247bd","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.53489184Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55732024-01-10T17:57:26.548Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.534962121Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363}
55742024-01-10T17:57:26.548Z{"msg":"f379de3e-653e-446a-900f-3594124247bd active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.535000478Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55752024-01-10T17:57:26.548Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.53510683Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"0","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55762024-01-10T17:57:26.548Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.535146232Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"0","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55772024-01-10T17:57:26.548Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.535183858Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"1","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55782024-01-10T17:57:26.548Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.535251351Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"1","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55792024-01-10T17:57:26.548Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.535319812Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"2","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55802024-01-10T17:57:26.548Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.535550186Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"2","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55812024-01-10T17:57:26.548Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.535656673Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55822024-01-10T17:57:26.548Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.535699243Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55832024-01-10T17:57:26.548Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.535764491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55842024-01-10T17:57:26.549Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.535833515Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55852024-01-10T17:57:26.549Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.53587459Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55862024-01-10T17:57:26.549Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.53594904Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55872024-01-10T17:57:26.549Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.53675868Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"1","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55882024-01-10T17:57:26.549Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.536869516Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"1","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55892024-01-10T17:57:26.550Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.536966446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"2","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55902024-01-10T17:57:26.550Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.537115279Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"2","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55912024-01-10T17:57:26.550Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.537827228Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"2","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55922024-01-10T17:57:26.550Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.537981599Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"1","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55932024-01-10T17:57:26.550Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.538064387Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"0","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55942024-01-10T17:57:26.551Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.538114047Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"0","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55952024-01-10T17:57:26.551Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.538824953Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"2","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55962024-01-10T17:57:26.551Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.538915078Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"2","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55972024-01-10T17:57:26.551Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.538996611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55982024-01-10T17:57:26.551Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.539092173Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"1","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
55992024-01-10T17:57:26.551Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.539164107Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"1","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56002024-01-10T17:57:26.551Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.539304862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56012024-01-10T17:57:26.551Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.539586239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"0","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56022024-01-10T17:57:26.551Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.54029624Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"0","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56032024-01-10T17:57:26.551Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540357817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"0","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56042024-01-10T17:57:26.553Z{"msg":"[0]R flush_numbers[0..12]: [91, 95, 71, 73, 71, 94, 94, 84, 96, 93, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540385531Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56052024-01-10T17:57:26.553Z{"msg":"[0]R generation[0..12]: [48, 50, 38, 40, 38, 50, 50, 44, 51, 49, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540408843Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56062024-01-10T17:57:26.553Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540451669Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56072024-01-10T17:57:26.553Z{"msg":"[1]R flush_numbers[0..12]: [91, 95, 71, 73, 71, 94, 94, 84, 96, 93, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540476674Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56082024-01-10T17:57:26.553Z{"msg":"[1]R generation[0..12]: [48, 50, 38, 40, 38, 50, 50, 44, 51, 49, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540521676Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56092024-01-10T17:57:26.553Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
56102024-01-10T17:57:26.553Z,"time":"2024-01-10T17:57:26.540546721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56112024-01-10T17:57:26.553Z{"msg":"[2]R flush_numbers[0..12]: [91, 95, 71, 73, 71, 94, 94, 84, 96, 93, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540589615Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56122024-01-10T17:57:26.553Z{"msg":"[2]R generation[0..12]: [48, 50, 38, 40, 38, 50, 50, 44, 51, 49, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540613874Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56132024-01-10T17:57:26.553Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540637407Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56142024-01-10T17:57:26.554Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540658683Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56152024-01-10T17:57:26.554Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540681636Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56162024-01-10T17:57:26.554Z{"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540702607Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56172024-01-10T17:57:26.554Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540723935Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"":"downstairs","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56182024-01-10T17:57:26.554Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.54076464Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56192024-01-10T17:57:26.554Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540786157Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"0","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56202024-01-10T17:57:26.554Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.5408089Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"1","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56212024-01-10T17:57:26.554Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.54083197Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"client":"2","":"downstairs","session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56222024-01-10T17:57:26.555Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540854171Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56232024-01-10T17:57:26.555Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540874216Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56242024-01-10T17:57:26.556Z{"msg":"f379de3e-653e-446a-900f-3594124247bd is now active with session: c58b81b0-2b8c-4001-a2c0-5a975c6395fa","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.54089537Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56252024-01-10T17:57:26.556Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.5409464Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363,"session_id":"c58b81b0-2b8c-4001-a2c0-5a975c6395fa"}
56262024-01-10T17:57:26.556Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:26.540968032Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1363}
56272024-01-10T17:57:26.556ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56282024-01-10T17:57:26.556ZLoading write count information from file "/tmp/test_repair_verify.data"
56292024-01-10T17:57:26.556ZRun Repair workload
56302024-01-10T17:57:26.556Z01/30 Write block 342 len 36864 data: 6 6 5 5 4 5 4 5 4
56312024-01-10T17:57:26.560Z02/30 Write block 292 len 40960 data: 8 7 6 5 5 6 8 8 7 6
56322024-01-10T17:57:26.560Z03/30 Read block 529 len 8192
56332024-01-10T17:57:26.560Z04/30 Read block 80 len 28672
56342024-01-10T17:57:26.560Z05/30 Read block 179 len 24576
56352024-01-10T17:57:26.560Z06/30 Read block 548 len 20480
56362024-01-10T17:57:26.560Z07/30 Read block 69 len 36864
56372024-01-10T17:57:26.560Z08/30 Read block 369 len 12288
56382024-01-10T17:57:26.564Z09/30 Write block 261 len 28672 data: 4 3 4 4 3 4 6
56392024-01-10T17:57:26.564Z10/30 Read block 16 len 4096
56402024-01-10T17:57:26.564Z11/30 Write block 510 len 12288 data: 7 7 8
56412024-01-10T17:57:26.564Z12/30 Flush
56422024-01-10T17:57:26.567Z13/30 Flush
56432024-01-10T17:57:26.570Z14/30 Write block 526 len 24576 data: 4 3 3 5 6 8
56442024-01-10T17:57:26.571Z15/30 Write block 240 len 20480 data: 6 5 5 4 7
56452024-01-10T17:57:26.571Z16/30 Read block 568 len 8192
56462024-01-10T17:57:26.575Z17/30 Read block 26 len 28672
56472024-01-10T17:57:26.577Z18/30 Write block 93 len 36864 data: 4 4 6 6 6 6 6 5 4
56482024-01-10T17:57:26.578Z19/30 Flush
56492024-01-10T17:57:26.581Z20/30 Write block 284 len 36864 data: 5 5 3 3 3 5 8 10 9
56502024-01-10T17:57:26.584Z21/30 Flush
56512024-01-10T17:57:26.587Z22/30 Write block 200 len 20480 data: 6 6 6 5 4
56522024-01-10T17:57:26.587Z23/30 Read block 242 len 4096
56532024-01-10T17:57:26.587Z24/30 Write block 422 len 8192 data: 8 7
56542024-01-10T17:57:26.587Z25/30 Read block 409 len 8192
56552024-01-10T17:57:26.592Z26/30 Write block 359 len 32768 data: 5 4 3 3 3 3 2 2
56562024-01-10T17:57:26.592Z27/30 Write block 571 len 36864 data: 4 3 4 6 6 7 6 5 5
56572024-01-10T17:57:26.592Z28/30 Read block 151 len 32768
56582024-01-10T17:57:26.595Z29/30 Read block 437 len 4096
56592024-01-10T17:57:26.598Z30/30 Read block 371 len 20480
56602024-01-10T17:57:26.601Z----------------------------------------------------------------
56612024-01-10T17:57:26.601Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
56622024-01-10T17:57:26.601ZStates: Active Active Active
56632024-01-10T17:57:26.601ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
56642024-01-10T17:57:26.602Z 1 Acked 1000 Write 9 Done Done Done false
56652024-01-10T17:57:26.602Z 2 Acked 1001 Write 10 Done Done Done false
56662024-01-10T17:57:26.602Z 3 Acked 1002 Read 2 Done Done Done false
56672024-01-10T17:57:26.602Z 4 Acked 1003 Read 7 Done Done Done false
56682024-01-10T17:57:26.603Z 5 Acked 1004 Read 6 Sent Done Done false
56692024-01-10T17:57:26.603Z 6 Acked 1005 Read 5 Sent Done Done false
56702024-01-10T17:57:26.603Z 7 Acked 1006 Read 9 Sent Done Done false
56712024-01-10T17:57:26.603Z 8 Acked 1007 Read 3 Sent Done Done false
56722024-01-10T17:57:26.603Z 9 Acked 1008 Write 7 Sent Done Done false
56732024-01-10T17:57:26.603Z 10 Acked 1009 Read 1 Sent Done Done false
56742024-01-10T17:57:26.603Z 11 Acked 1010 Write 3 Sent Done Done false
56752024-01-10T17:57:26.603Z 12 Acked 1011 Flush 0 Sent Done Done false
56762024-01-10T17:57:26.603Z 13 Acked 1012 Flush 0 Sent Done Done false
56772024-01-10T17:57:26.603Z 14 Acked 1013 Write 6 Sent Done Done false
56782024-01-10T17:57:26.603Z 15 Acked 1014 Write 5 Sent Done Done false
56792024-01-10T17:57:26.603Z 16 Acked 1015 Read 2 Sent Done Done false
56802024-01-10T17:57:26.603Z 17 Acked 1016 Read 7 Sent Done Done false
56812024-01-10T17:57:26.603Z 18 Acked 1017 Write 9 Sent Done Done false
56822024-01-10T17:57:26.603Z 19 Acked 1018 Flush 0 Sent Done Done false
56832024-01-10T17:57:26.603Z 20 Acked 1019 Write 9 Sent Done Done false
56842024-01-10T17:57:26.603Z 21 Acked 1020 Flush 0 Sent Done Done false
56852024-01-10T17:57:26.603Z 22 Acked 1021 Write 5 Sent Done Done false
56862024-01-10T17:57:26.603Z 23 Acked 1022 Read 1 Sent Done Done false
56872024-01-10T17:57:26.603Z 24 Acked 1023 Write 2 Sent Done Done false
56882024-01-10T17:57:26.603Z 25 Acked 1024 Read 2 Sent Done Done false
56892024-01-10T17:57:26.604Z 26 Acked 1025 Write 8 Sent Done Done false
56902024-01-10T17:57:26.604Z 27 Acked 1026 Write 9 Sent Done Done false
56912024-01-10T17:57:26.604Z 28 Acked 1027 Read 8 Sent Done Done false
56922024-01-10T17:57:26.604Z 29 Acked 1028 Read 1 Sent Done Done false
56932024-01-10T17:57:26.604Z 30 Acked 1029 Read 5 Sent Sent Done false
56942024-01-10T17:57:26.604Z STATES DS:0 DS:1 DS:2 TOTAL
56952024-01-10T17:57:26.604Z New 0 0 0 0
56962024-01-10T17:57:26.604Z Sent 26 1 0 27
56972024-01-10T17:57:26.605Z Done 4 29 30 63
56982024-01-10T17:57:26.605Z Skipped 0 0 0 0
56992024-01-10T17:57:26.605Z Error 0 0 0 0
57002024-01-10T17:57:26.605ZLast Flush: 0 1020 1020
57012024-01-10T17:57:26.605ZDownstairs last five completed:
57022024-01-10T17:57:26.605ZUpstairs last five completed: 30 29 28 27 26
57032024-01-10T17:57:26.605ZWrote out file "/tmp/test_repair_verify.data"
57042024-01-10T17:57:26.609Z
57052024-01-10T17:57:26.612ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57062024-01-10T17:57:26.639ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57072024-01-10T17:57:26.639Z 0 000-019 48 48 48 91 91 91 F F F
57082024-01-10T17:57:26.639Z 1 020-039 50 50 50 95 95 95 F F F
57092024-01-10T17:57:26.639Z 2 040-059 38 38 38 71 71 71 F F F
57102024-01-10T17:57:26.639Z 3 060-079 40 40 40 73 73 73 F F F
57112024-01-10T17:57:26.639Z 4 080-099 38 52 52 71 99 99 F F F <---
57122024-01-10T17:57:26.643Z 5 100-119 50 52 52 94 99 99 F F F <---
57132024-01-10T17:57:26.643Z 6 120-139 50 50 50 94 94 94 F F F
57142024-01-10T17:57:26.643Z 7 140-159 44 44 44 84 84 84 F F F
57152024-01-10T17:57:26.643Z 8 160-179 51 51 51 96 96 96 F F F
57162024-01-10T17:57:26.643Z 9 180-199 49 49 49 93 93 93 F F F
57172024-01-10T17:57:26.643Z 10 200-219 51 51 51 96 96 96 F T T <---
57182024-01-10T17:57:26.644Z 11 220-239 51 51 51 96 96 96 F F F
57192024-01-10T17:57:26.644Z 12 240-259 51 52 52 96 99 99 F F F <---
57202024-01-10T17:57:26.644Z 13 260-279 50 52 52 95 97 97 F F F <---
57212024-01-10T17:57:26.644Z 14 280-299 46 52 52 88 100 100 T F F <---
57222024-01-10T17:57:26.644Z 15 300-319 51 52 52 96 97 97 T F F <---
57232024-01-10T17:57:26.644Z 16 320-339 37 37 37 70 70 70 F F F
57242024-01-10T17:57:26.644Z 17 340-359 50 52 52 94 97 97 T T T <---
57252024-01-10T17:57:26.644Z 18 360-379 44 44 44 85 85 85 F T T <---
57262024-01-10T17:57:26.644Z 19 380-399 42 42 42 77 77 77 F F F
57272024-01-10T17:57:26.644Z 20 400-419 47 47 47 89 89 89 F F F
57282024-01-10T17:57:26.644Z 21 420-439 47 47 47 89 89 89 F T T <---
57292024-01-10T17:57:26.644Z 22 440-459 50 50 50 95 95 95 F F F
57302024-01-10T17:57:26.644Z 23 460-479 49 49 49 93 93 93 F F F
57312024-01-10T17:57:26.644Z 24 480-499 51 51 51 96 96 96 F F F
57322024-01-10T17:57:26.644Z 25 500-519 45 52 52 86 97 97 F F F <---
57332024-01-10T17:57:26.644Z 26 520-539 50 52 52 95 99 99 F F F <---
57342024-01-10T17:57:26.644Z 27 540-559 47 47 47 89 89 89 F F F
57352024-01-10T17:57:26.644Z 28 560-579 50 50 50 95 95 95 F T T <---
57362024-01-10T17:57:26.644Z 29 580-599 50 50 50 95 95 95 F F F
57372024-01-10T17:57:26.644ZMax gen: 52, Max flush: 100
57382024-01-10T17:57:26.644ZError: Difference in extent metadata found!
57392024-01-10T17:57:26.647ZA Difference in extent metadata is expected here
57402024-01-10T17:57:26.648ZOn loop 26
57412024-01-10T17:57:26.648Z
57422024-01-10T17:57:26.656ZVerifying data now
57432024-01-10T17:57:32.922ZNo change after verify
57442024-01-10T17:57:32.922ZLoop: 26 Downstairs dump after verify (and repair):
57452024-01-10T17:57:32.945ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57462024-01-10T17:57:32.945Z 0 000-019 48 48 48 91 91 91 F F F
57472024-01-10T17:57:32.945Z 1 020-039 50 50 50 95 95 95 F F F
57482024-01-10T17:57:32.945Z 2 040-059 38 38 38 71 71 71 F F F
57492024-01-10T17:57:32.945Z 3 060-079 40 40 40 73 73 73 F F F
57502024-01-10T17:57:32.945Z 4 080-099 52 52 52 99 99 99 F F F
57512024-01-10T17:57:32.945Z 5 100-119 52 52 52 99 99 99 F F F
57522024-01-10T17:57:32.945Z 6 120-139 50 50 50 94 94 94 F F F
57532024-01-10T17:57:32.945Z 7 140-159 44 44 44 84 84 84 F F F
57542024-01-10T17:57:32.949Z 8 160-179 51 51 51 96 96 96 F F F
57552024-01-10T17:57:32.949Z 9 180-199 49 49 49 93 93 93 F F F
57562024-01-10T17:57:32.949Z 10 200-219 53 53 53 101 101 101 F F F
57572024-01-10T17:57:32.949Z 11 220-239 51 51 51 96 96 96 F F F
57582024-01-10T17:57:32.950Z 12 240-259 52 52 52 99 99 99 F F F
57592024-01-10T17:57:32.950Z 13 260-279 52 52 52 97 97 97 F F F
57602024-01-10T17:57:32.950Z 14 280-299 52 52 52 100 100 100 F F F
57612024-01-10T17:57:32.950Z 15 300-319 52 52 52 97 97 97 F F F
57622024-01-10T17:57:32.950Z 16 320-339 37 37 37 70 70 70 F F F
57632024-01-10T17:57:32.950Z 17 340-359 53 53 53 101 101 101 F F F
57642024-01-10T17:57:32.950Z 18 360-379 53 53 53 101 101 101 F F F
57652024-01-10T17:57:32.950Z 19 380-399 42 42 42 77 77 77 F F F
57662024-01-10T17:57:32.950Z 20 400-419 47 47 47 89 89 89 F F F
57672024-01-10T17:57:32.950Z 21 420-439 53 53 53 101 101 101 F F F
57682024-01-10T17:57:32.950Z 22 440-459 50 50 50 95 95 95 F F F
57692024-01-10T17:57:32.950Z 23 460-479 49 49 49 93 93 93 F F F
57702024-01-10T17:57:32.950Z 24 480-499 51 51 51 96 96 96 F F F
57712024-01-10T17:57:32.950Z 25 500-519 52 52 52 97 97 97 F F F
57722024-01-10T17:57:32.950Z 26 520-539 52 52 52 99 99 99 F F F
57732024-01-10T17:57:32.950Z 27 540-559 47 47 47 89 89 89 F F F
57742024-01-10T17:57:32.950Z 28 560-579 53 53 53 101 101 101 F F F
57752024-01-10T17:57:32.950Z 29 580-599 50 50 50 95 95 95 F F F
57762024-01-10T17:57:32.950ZMax gen: 53, Max flush: 101
57772024-01-10T17:57:32.954Z
57782024-01-10T17:57:32.954ZBegin loop 27
57792024-01-10T17:57:32.954ZDownstairs to restart: 2
57802024-01-10T17:57:32.984Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.984479504Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371}
57812024-01-10T17:57:32.984Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.984962443Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371}
57822024-01-10T17:57:32.984Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.98499159Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371}
57832024-01-10T17:57:32.984Z{"msg":"Crucible cae93da6-f0c4-41a2-ab8c-4935e411df75 has session id: 2697b241-5360-4fed-ae04-8c1a784dd8c4","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.985014839Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57842024-01-10T17:57:32.989Z{"msg":"Upstairs opts: Upstairs UUID: cae93da6-f0c4-41a2-ab8c-4935e411df75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible",Crucible runtime is spawned
57852024-01-10T17:57:32.989Z"level":30,"time":"2024-01-10T17:57:32.985248737Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57862024-01-10T17:57:32.989Z{"msg":"Crucible stats registered with UUID: cae93da6-f0c4-41a2-ab8c-4935e411df75","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.985303973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57872024-01-10T17:57:32.989Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.985579226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371}
57882024-01-10T17:57:32.989Z{"msg":"cae93da6-f0c4-41a2-ab8c-4935e411df75 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.985653355Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57892024-01-10T17:57:32.989Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.985695869Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"0","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57902024-01-10T17:57:32.989Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.985802028Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"0","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57912024-01-10T17:57:32.989Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.985849528Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"1","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57922024-01-10T17:57:32.989Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.985922669Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"1","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57932024-01-10T17:57:32.989Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.985965312Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"2","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57942024-01-10T17:57:32.989Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.986038995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"2","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57952024-01-10T17:57:32.989Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.986099833Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57962024-01-10T17:57:32.989Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.986169165Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57972024-01-10T17:57:32.990Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.986237682Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57982024-01-10T17:57:32.990Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.986308498Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
57992024-01-10T17:57:32.990Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.986403379Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58002024-01-10T17:57:32.992Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.98644925Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58012024-01-10T17:57:32.992Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.987764861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"1","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58022024-01-10T17:57:32.992Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.987851638Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"1","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58032024-01-10T17:57:32.992Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.987878886Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"0","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58042024-01-10T17:57:32.993Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.98804114Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"0","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58052024-01-10T17:57:32.999Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.988963236Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"1","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58062024-01-10T17:57:32.999Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.98918487Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"0","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58072024-01-10T17:57:32.999Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.989807924Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"2","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58082024-01-10T17:57:32.999Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.989944119Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"2","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58092024-01-10T17:57:32.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.990081562Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"1","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58102024-01-10T17:57:32.999Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.990160204Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"1","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58112024-01-10T17:57:33.000Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.990234358Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58122024-01-10T17:57:33.000Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.990364618Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"0","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58132024-01-10T17:57:33.000Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.990461484Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"0","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58142024-01-10T17:57:33.000Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.990488286Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58152024-01-10T17:57:33.000Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.990807807Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"2","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58162024-01-10T17:57:33.000Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.99149091Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"2","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58172024-01-10T17:57:33.000Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991557222Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"2","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58182024-01-10T17:57:33.000Z{"msg":"[0]R flush_numbers[0..12]: [91, 95, 71, 73, 99, 99, 94, 84, 96, 93, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991583232Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58192024-01-10T17:57:33.000Z{"msg":"[0]R generation[0..12]: [48, 50, 38, 40, 52, 52, 50, 44, 51, 49, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991606418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58202024-01-10T17:57:33.000Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991630064Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58212024-01-10T17:57:33.000Z{"msg":"[1]R flush_numbers[0..12]: [91, 95, 71, 73, 99, 99, 94, 84, 96, 93, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991673379Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58222024-01-10T17:57:33.000Z{"msg":"[1]R generation[0..12]: [48, 50, 38, 40, 52, 52, 50, 44, 51, 49, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991697606Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58232024-01-10T17:57:33.000Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991721559Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58242024-01-10T17:57:33.000Z{"msg":"[2]R flush_numbers[0..12]: [91, 95, 71, 73, 99, 99, 94, 84, 96, 93, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991744705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58252024-01-10T17:57:33.000Z{"msg":"[2]R generation[0..12]: [48, 50, 38, 40, 52, 52, 50, 44, 51, 49, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991767277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58262024-01-10T17:57:33.000Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991813454Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58272024-01-10T17:57:33.000Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991836633Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58282024-01-10T17:57:33.002Z{"msg":"Generation requested: 54 >= found:54","v":0,Wait for a query_work_queue command to finish before sending IO
58292024-01-10T17:57:33.002Z"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991863607Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58302024-01-10T17:57:33.002Z{"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991885251Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58312024-01-10T17:57:33.002Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991906447Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"":"downstairs","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58322024-01-10T17:57:33.002Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991946278Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58332024-01-10T17:57:33.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991967236Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"0","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58342024-01-10T17:57:33.003Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.991988959Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"1","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58352024-01-10T17:57:33.003Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.992010456Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"client":"2","":"downstairs","session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58362024-01-10T17:57:33.003Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.992031795Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58372024-01-10T17:57:33.003Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.99205303Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58382024-01-10T17:57:33.003Z{"msg":"cae93da6-f0c4-41a2-ab8c-4935e411df75 is now active with session: 2697b241-5360-4fed-ae04-8c1a784dd8c4","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.9920744Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58392024-01-10T17:57:33.003Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.992093812Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371,"session_id":"2697b241-5360-4fed-ae04-8c1a784dd8c4"}
58402024-01-10T17:57:33.003Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:32.992131957Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1371}
58412024-01-10T17:57:33.004ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58422024-01-10T17:57:33.005ZLoading write count information from file "/tmp/test_repair_verify.data"
58432024-01-10T17:57:33.005ZRun Repair workload
58442024-01-10T17:57:33.005Z01/30 Write block 273 len 12288 data: 4 6 7
58452024-01-10T17:57:33.005Z02/30 Write block 587 len 20480 data: 4 4 5 5 5
58462024-01-10T17:57:33.005Z03/30 Read block 255 len 8192
58472024-01-10T17:57:33.012Z04/30 Write block 297 len 20480 data: 7 9 9 8 7
58482024-01-10T17:57:33.012Z05/30 Read block 588 len 20480
58492024-01-10T17:57:33.012Z06/30 Write block 302 len 40960 data: 5 6 5 5 5 4 3 3 2 3
58502024-01-10T17:57:33.012Z07/30 Read block 241 len 28672
58512024-01-10T17:57:33.012Z08/30 Flush
58522024-01-10T17:57:33.016Z09/30 Write block 177 len 28672 data: 6 6 5 7 6 6 6
58532024-01-10T17:57:33.016Z10/30 Read block 142 len 16384
58542024-01-10T17:57:33.016Z11/30 Read block 88 len 28672
58552024-01-10T17:57:33.023Z12/30 Write block 492 len 28672 data: 7 7 8 11 9 8 8
58562024-01-10T17:57:33.023Z13/30 Read block 143 len 8192
58572024-01-10T17:57:33.023Z14/30 Write block 539 len 28672 data: 5 6 6 4 5 6 7
58582024-01-10T17:57:33.023Z15/30 Read block 494 len 12288
58592024-01-10T17:57:33.023Z16/30 Read block 119 len 12288
58602024-01-10T17:57:33.027Z17/30 Write block 425 len 20480 data: 5 4 4 4 4
58612024-01-10T17:57:33.027Z18/30 Flush
58622024-01-10T17:57:33.030Z19/30 Write block 410 len 12288 data: 6 7 6
58632024-01-10T17:57:33.030Z20/30 Read block 192 len 28672
58642024-01-10T17:57:33.034Z21/30 Write block 26 len 40960 data: 6 6 5 6 9 7 7 6 4 5
58652024-01-10T17:57:33.034Z22/30 Read block 371 len 20480
58662024-01-10T17:57:33.038Z23/30 Read block 559 len 28672
58672024-01-10T17:57:33.038Z24/30 Read block 403 len 4096
58682024-01-10T17:57:33.041Z25/30 Write block 279 len 40960 data: 5 5 5 5 6 6 6 4 4 4
58692024-01-10T17:57:33.041Z26/30 Write block 386 len 4096 data: 3
58702024-01-10T17:57:33.041Z27/30 Read block 470 len 16384
58712024-01-10T17:57:33.045Z28/30 Write block 493 len 32768 data: 8 9 12 10 9 9 9 6
58722024-01-10T17:57:33.045Z29/30 Write block 23 len 4096 data: 6
58732024-01-10T17:57:33.049Z30/30 Write block 36 len 32768 data: 5 4 5 3 4 5 5 4
58742024-01-10T17:57:33.049Z----------------------------------------------------------------
58752024-01-10T17:57:33.049Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
58762024-01-10T17:57:33.049ZStates: Active Active Active
58772024-01-10T17:57:33.049ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
58782024-01-10T17:57:33.049Z 1 Acked 1000 Write 3 Done Done Done false
58792024-01-10T17:57:33.049Z 2 Acked 1001 Write 5 Done Done Done false
58802024-01-10T17:57:33.050Z 3 Acked 1002 Read 2 Done Done Done false
58812024-01-10T17:57:33.050Z 4 Acked 1003 Write 5 Done Done Sent false
58822024-01-10T17:57:33.050Z 5 Acked 1004 Read 5 Done Done Sent false
58832024-01-10T17:57:33.053Z 6 Acked 1005 Write 10 Done Done Sent false
58842024-01-10T17:57:33.053Z 7 Acked 1006 Read 7 Done Done Sent false
58852024-01-10T17:57:33.054Z 8 Acked 1007 Flush 0 Done Done Sent false
58862024-01-10T17:57:33.054Z 9 Acked 1008 Write 7 Done Done Sent false
58872024-01-10T17:57:33.054Z 10 Acked 1009 Read 4 Done Done Sent false
58882024-01-10T17:57:33.054Z 11 Acked 1010 Read 7 Done Done Sent false
58892024-01-10T17:57:33.055Z 12 Acked 1011 Write 7 Done Done Sent false
58902024-01-10T17:57:33.055Z 13 Acked 1012 Read 2 Done Done Sent false
58912024-01-10T17:57:33.055Z 14 Acked 1013 Write 7 Done Done Sent false
58922024-01-10T17:57:33.055Z 15 Acked 1014 Read 3 Done Done Sent false
58932024-01-10T17:57:33.055Z 16 Acked 1015 Read 3 Done Done Sent false
58942024-01-10T17:57:33.055Z 17 Acked 1016 Write 5 Done Done Sent false
58952024-01-10T17:57:33.055Z 18 Acked 1017 Flush 0 Done Done Sent false
58962024-01-10T17:57:33.055Z 19 Acked 1018 Write 3 Done Done Sent false
58972024-01-10T17:57:33.055Z 20 Acked 1019 Read 7 Done Done Sent false
58982024-01-10T17:57:33.055Z 21 Acked 1020 Write 10 Done Done Sent false
58992024-01-10T17:57:33.055Z 22 Acked 1021 Read 5 Done Done Sent false
59002024-01-10T17:57:33.055Z 23 Acked 1022 Read 7 Done Done Sent false
59012024-01-10T17:57:33.055Z 24 Acked 1023 Read 1 Done Done Sent false
59022024-01-10T17:57:33.055Z 25 Acked 1024 Write 10 Done Done Sent false
59032024-01-10T17:57:33.055Z 26 Acked 1025 Write 1 Done Done Sent false
59042024-01-10T17:57:33.055Z 27 Acked 1026 Read 4 Done Done Sent false
59052024-01-10T17:57:33.055Z 28 Acked 1027 Write 8 Done Done Sent false
59062024-01-10T17:57:33.055Z 29 Acked 1028 Write 1 Sent Sent Sent false
59072024-01-10T17:57:33.055Z 30 Acked 1029 Write 8 Sent Sent Sent false
59082024-01-10T17:57:33.055Z STATES DS:0 DS:1 DS:2 TOTAL
59092024-01-10T17:57:33.055Z New 0 0 0 0
59102024-01-10T17:57:33.055Z Sent 2 2 27 31
59112024-01-10T17:57:33.055Z Done 28 28 3 59
59122024-01-10T17:57:33.055Z Skipped 0 0 0 0
59132024-01-10T17:57:33.056Z Error 0 0 0 0
59142024-01-10T17:57:33.056ZLast Flush: 1017 1017 0
59152024-01-10T17:57:33.056ZDownstairs last five completed:
59162024-01-10T17:57:33.056ZUpstairs last five completed: 30 29 28 27 26
59172024-01-10T17:57:33.056ZWrote out file "/tmp/test_repair_verify.data"
59182024-01-10T17:57:33.060Z
59192024-01-10T17:57:33.063ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59202024-01-10T17:57:33.097ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59212024-01-10T17:57:33.097Z 0 000-019 48 48 48 91 91 91 F F F
59222024-01-10T17:57:33.097Z 1 020-039 50 50 50 95 95 95 T T F <---
59232024-01-10T17:57:33.097Z 2 040-059 38 38 38 71 71 71 T T F <---
59242024-01-10T17:57:33.097Z 3 060-079 40 40 40 73 73 73 F F F
59252024-01-10T17:57:33.097Z 4 080-099 52 52 52 99 99 99 F F F
59262024-01-10T17:57:33.097Z 5 100-119 52 52 52 99 99 99 F F F
59272024-01-10T17:57:33.097Z 6 120-139 50 50 50 94 94 94 F F F
59282024-01-10T17:57:33.097Z 7 140-159 44 44 44 84 84 84 F F F
59292024-01-10T17:57:33.098Z 8 160-179 54 54 51 103 103 96 F F F <---
59302024-01-10T17:57:33.098Z 9 180-199 54 54 49 103 103 93 F F F <---
59312024-01-10T17:57:33.098Z 10 200-219 53 53 53 101 101 101 F F F
59322024-01-10T17:57:33.098Z 11 220-239 51 51 51 96 96 96 F F F
59332024-01-10T17:57:33.098Z 12 240-259 52 52 52 99 99 99 F F F
59342024-01-10T17:57:33.098Z 13 260-279 54 54 52 102 102 97 T T T <---
59352024-01-10T17:57:33.098Z 14 280-299 54 54 52 102 102 100 T T F <---
59362024-01-10T17:57:33.098Z 15 300-319 54 54 52 102 102 97 F F F <---
59372024-01-10T17:57:33.105Z 16 320-339 37 37 37 70 70 70 F F F
59382024-01-10T17:57:33.105Z 17 340-359 53 53 53 101 101 101 F F F
59392024-01-10T17:57:33.105Z 18 360-379 53 53 53 101 101 101 F F F
59402024-01-10T17:57:33.106Z 19 380-399 42 42 42 77 77 77 T T F <---
59412024-01-10T17:57:33.106Z 20 400-419 47 47 47 89 89 89 T T F <---
59422024-01-10T17:57:33.106Z 21 420-439 54 54 53 103 103 101 F F F <---
59432024-01-10T17:57:33.106Z 22 440-459 50 50 50 95 95 95 F F F
59442024-01-10T17:57:33.106Z 23 460-479 49 49 49 93 93 93 F F F
59452024-01-10T17:57:33.106Z 24 480-499 54 54 51 103 103 96 T T F <---
59462024-01-10T17:57:33.106Z 25 500-519 52 52 52 97 97 97 T T F <---
59472024-01-10T17:57:33.106Z 26 520-539 54 54 52 103 103 99 F F F <---
59482024-01-10T17:57:33.106Z 27 540-559 54 54 47 103 103 89 F F F <---
59492024-01-10T17:57:33.106Z 28 560-579 53 53 53 101 101 101 F F F
59502024-01-10T17:57:33.106Z 29 580-599 54 54 50 102 102 95 F F T <---
59512024-01-10T17:57:33.106ZMax gen: 54, Max flush: 103
59522024-01-10T17:57:33.106ZError: Difference in extent metadata found!
59532024-01-10T17:57:33.106ZA Difference in extent metadata is expected here
59542024-01-10T17:57:33.106ZOn loop 27
59552024-01-10T17:57:33.107Z
59562024-01-10T17:57:33.107ZVerifying data now
59572024-01-10T17:57:39.546ZNo change after verify
59582024-01-10T17:57:39.547ZLoop: 27 Downstairs dump after verify (and repair):
59592024-01-10T17:57:39.568ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59602024-01-10T17:57:39.569Z 0 000-019 48 48 48 91 91 91 F F F
59612024-01-10T17:57:39.569Z 1 020-039 55 55 55 104 104 104 F F F
59622024-01-10T17:57:39.569Z 2 040-059 55 55 55 104 104 104 F F F
59632024-01-10T17:57:39.569Z 3 060-079 40 40 40 73 73 73 F F F
59642024-01-10T17:57:39.569Z 4 080-099 52 52 52 99 99 99 F F F
59652024-01-10T17:57:39.569Z 5 100-119 52 52 52 99 99 99 F F F
59662024-01-10T17:57:39.569Z 6 120-139 50 50 50 94 94 94 F F F
59672024-01-10T17:57:39.569Z 7 140-159 44 44 44 84 84 84 F F F
59682024-01-10T17:57:39.569Z 8 160-179 54 54 54 103 103 103 F F F
59692024-01-10T17:57:39.569Z 9 180-199 54 54 54 103 103 103 F F F
59702024-01-10T17:57:39.569Z 10 200-219 53 53 53 101 101 101 F F F
59712024-01-10T17:57:39.569Z 11 220-239 51 51 51 96 96 96 F F F
59722024-01-10T17:57:39.569Z 12 240-259 52 52 52 99 99 99 F F F
59732024-01-10T17:57:39.569Z 13 260-279 55 55 55 104 104 104 F F F
59742024-01-10T17:57:39.569Z 14 280-299 55 55 55 104 104 104 F F F
59752024-01-10T17:57:39.572Z 15 300-319 54 54 54 102 102 102 F F F
59762024-01-10T17:57:39.573Z 16 320-339 37 37 37 70 70 70 F F F
59772024-01-10T17:57:39.574Z 17 340-359 53 53 53 101 101 101 F F F
59782024-01-10T17:57:39.574Z 18 360-379 53 53 53 101 101 101 F F F
59792024-01-10T17:57:39.574Z 19 380-399 55 55 55 104 104 104 F F F
59802024-01-10T17:57:39.574Z 20 400-419 55 55 55 104 104 104 F F F
59812024-01-10T17:57:39.574Z 21 420-439 54 54 54 103 103 103 F F F
59822024-01-10T17:57:39.575Z 22 440-459 50 50 50 95 95 95 F F F
59832024-01-10T17:57:39.575Z 23 460-479 49 49 49 93 93 93 F F F
59842024-01-10T17:57:39.575Z 24 480-499 55 55 55 104 104 104 F F F
59852024-01-10T17:57:39.575Z 25 500-519 55 55 55 104 104 104 F F F
59862024-01-10T17:57:39.575Z 26 520-539 54 54 54 103 103 103 F F F
59872024-01-10T17:57:39.575Z 27 540-559 54 54 54 103 103 103 F F F
59882024-01-10T17:57:39.575Z 28 560-579 53 53 53 101 101 101 F F F
59892024-01-10T17:57:39.575Z 29 580-599 54 54 54 102 102 102 F F F
59902024-01-10T17:57:39.575ZMax gen: 55, Max flush: 104
59912024-01-10T17:57:39.578Z
59922024-01-10T17:57:39.579ZBegin loop 28
59932024-01-10T17:57:39.579ZDownstairs to restart: 2
59942024-01-10T17:57:39.608Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.60832679Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379}
59952024-01-10T17:57:39.608Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.608866695Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379}
59962024-01-10T17:57:39.608Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.608899152Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379}
59972024-01-10T17:57:39.613Z{"msg":"Crucible 44e6b4a9-0bed-4acd-a6c8-2829dc304555 has session id: 11f4422e-3981-4689-a310-dd81f7883dd1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.60895893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
59982024-01-10T17:57:39.613Z{"msg":"Upstairs opts: Upstairs UUID: 44e6b4a9-0bed-4acd-a6c8-2829dc304555, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.60902684Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
59992024-01-10T17:57:39.613Z{"msg":"Crucible stats registered with UUID: 44e6b4a9-0bed-4acd-a6c8-2829dc304555","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.609074156Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60002024-01-10T17:57:39.613ZCrucible runtime is spawned
60012024-01-10T17:57:39.613Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.609502149Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379}
60022024-01-10T17:57:39.613Z{"msg":"44e6b4a9-0bed-4acd-a6c8-2829dc304555 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.609583195Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60032024-01-10T17:57:39.613Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.609623736Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"0","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60042024-01-10T17:57:39.613Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.609719941Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"0","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60052024-01-10T17:57:39.613Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.609764119Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"1","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60062024-01-10T17:57:39.613Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.609802556Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"1","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60072024-01-10T17:57:39.613Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.609839234Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"2","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60082024-01-10T17:57:39.613Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.609923213Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"2","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60092024-01-10T17:57:39.613Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.61000538Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60102024-01-10T17:57:39.614Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.610077915Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60112024-01-10T17:57:39.614Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.610200827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60122024-01-10T17:57:39.614Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.610245924Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60132024-01-10T17:57:39.614Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.610321132Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60142024-01-10T17:57:39.614Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.610361983Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60152024-01-10T17:57:39.614Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.611559656Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"1","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60162024-01-10T17:57:39.614Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.61164337Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"1","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60172024-01-10T17:57:39.614Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.611741311Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"0","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60182024-01-10T17:57:39.614Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.611788277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"0","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60192024-01-10T17:57:39.621Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.612501031Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"1","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60202024-01-10T17:57:39.622Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.612672004Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"0","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60212024-01-10T17:57:39.627Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.613119814Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"2","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60222024-01-10T17:57:39.627Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.613295898Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"2","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60232024-01-10T17:57:39.628Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.613395787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"1","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60242024-01-10T17:57:39.628Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.613440841Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"1","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60252024-01-10T17:57:39.628Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.613483862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60262024-01-10T17:57:39.628Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.613594986Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"0","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60272024-01-10T17:57:39.628Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.613659668Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"0","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60282024-01-10T17:57:39.628Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.613691081Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60292024-01-10T17:57:39.628Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.614163145Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"2","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60302024-01-10T17:57:39.628Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.614848775Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"2","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60312024-01-10T17:57:39.628Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.614939307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"2","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60322024-01-10T17:57:39.628Z{"msg":"[0]R flush_numbers[0..12]: [91, 104, 104, 73, 99, 99, 94, 84, 103, 103, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.614966905Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60332024-01-10T17:57:39.628Z{"msg":"[0]R generation[0..12]: [48, 55, 55, 40, 52, 52, 50, 44, 54, 54, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.614989924Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60342024-01-10T17:57:39.628Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615014796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60352024-01-10T17:57:39.628Z{"msg":"[1]R flush_numbers[0..12]: [91, 104, 104, 73, 99, 99, 94, 84, 103, 103, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615037874Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60362024-01-10T17:57:39.628Z{"msg":"[1]R generation[0..12]: [48, 55, 55, 40, 52, 52, 50, 44, 54, 54, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615061782Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60372024-01-10T17:57:39.628Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615085735Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60382024-01-10T17:57:39.630Z{"msg":"[2]R flush_numbers[0..12]: [91, 104, 104, 73, 99, 99, 94, 84, 103, 103, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615109926Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60392024-01-10T17:57:39.630Z{"msg":"[2]R generation[0..12]: [48, 55, 55, 40, 52, 52, 50, 44, 54, 54, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615132909Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60402024-01-10T17:57:39.630Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615157184Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60412024-01-10T17:57:39.630Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615178809Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"Wait for a query_work_queue command to finish before sending IO
60422024-01-10T17:57:39.630Zdownstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60432024-01-10T17:57:39.630Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615204143Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60442024-01-10T17:57:39.630Z{"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615226313Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60452024-01-10T17:57:39.630Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615247598Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"":"downstairs","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60462024-01-10T17:57:39.630Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615269147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60472024-01-10T17:57:39.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615309238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"0","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60482024-01-10T17:57:39.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615332272Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"1","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60492024-01-10T17:57:39.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615353901Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"client":"2","":"downstairs","session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60502024-01-10T17:57:39.630Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615461079Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60512024-01-10T17:57:39.630Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615502431Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60522024-01-10T17:57:39.630Z{"msg":"44e6b4a9-0bed-4acd-a6c8-2829dc304555 is now active with session: 11f4422e-3981-4689-a310-dd81f7883dd1","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615561657Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60532024-01-10T17:57:39.630Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.615600357Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379,"session_id":"11f4422e-3981-4689-a310-dd81f7883dd1"}
60542024-01-10T17:57:39.630Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:39.61566023Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1379}
60552024-01-10T17:57:39.630ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
60562024-01-10T17:57:39.630ZLoading write count information from file "/tmp/test_repair_verify.data"
60572024-01-10T17:57:39.630ZRun Repair workload
60582024-01-10T17:57:39.630Z01/30 Write block 54 len 32768 data: 6 6 5 4 4 4 4 3
60592024-01-10T17:57:39.630Z02/30 Read block 163 len 12288
60602024-01-10T17:57:39.630Z03/30 Write block 398 len 24576 data: 4 6 4 5 4 4
60612024-01-10T17:57:39.631Z04/30 Read block 425 len 36864
60622024-01-10T17:57:39.631Z05/30 Write block 245 len 20480 data: 6 4 3 3 4
60632024-01-10T17:57:39.635Z06/30 Read block 80 len 8192
60642024-01-10T17:57:39.635Z07/30 Read block 216 len 28672
60652024-01-10T17:57:39.635Z08/30 Read block 288 len 16384
60662024-01-10T17:57:39.635Z09/30 Read block 197 len 28672
60672024-01-10T17:57:39.638Z10/30 Write block 143 len 36864 data: 5 5 4 4 4 4 4 4 3
60682024-01-10T17:57:39.638Z11/30 Read block 303 len 4096
60692024-01-10T17:57:39.638Z12/30 Read block 365 len 16384
60702024-01-10T17:57:39.642Z13/30 Read block 364 len 40960
60712024-01-10T17:57:39.645Z14/30 Read block 281 len 20480
60722024-01-10T17:57:39.648Z15/30 Flush
60732024-01-10T17:57:39.651Z16/30 Flush
60742024-01-10T17:57:39.651Z17/30 Write block 273 len 28672 data: 5 7 8 7 6 5 6
60752024-01-10T17:57:39.651Z18/30 Write block 488 len 16384 data: 5 4 6 6
60762024-01-10T17:57:39.654Z19/30 Write block 561 len 36864 data: 5 6 7 7 7 5 5 6 6
60772024-01-10T17:57:39.655Z20/30 Read block 299 len 40960
60782024-01-10T17:57:39.657Z21/30 Write block 403 len 20480 data: 5 5 5 6 7
60792024-01-10T17:57:39.660Z22/30 Flush
60802024-01-10T17:57:39.663Z23/30 Write block 405 len 8192 data: 6 7
60812024-01-10T17:57:39.663Z24/30 Write block 489 len 20480 data: 5 7 7 8 9
60822024-01-10T17:57:39.666Z25/30 Write block 439 len 12288 data: 5 5 7
60832024-01-10T17:57:39.666Z26/30 Write block 525 len 40960 data: 2 5 4 4 6 7 9 7 7 6
60842024-01-10T17:57:39.666Z27/30 Read block 272 len 20480
60852024-01-10T17:57:39.670Z28/30 Read block 400 len 32768
60862024-01-10T17:57:39.672Z29/30 Write block 104 len 40960 data: 2 3 3 2 3 4 4 5 6 6
60872024-01-10T17:57:39.677Z30/30 Write block 449 len 20480 data: 6 7 4 4 5
60882024-01-10T17:57:39.677Z----------------------------------------------------------------
60892024-01-10T17:57:39.677Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
60902024-01-10T17:57:39.678ZStates: Active Active Active
60912024-01-10T17:57:39.678ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
60922024-01-10T17:57:39.678Z 1 Acked 1000 Write 8 Done Done Done false
60932024-01-10T17:57:39.678Z 2 Acked 1001 Read 3 Done Done Done false
60942024-01-10T17:57:39.678Z 3 Acked 1002 Write 6 Done Done Done false
60952024-01-10T17:57:39.678Z 4 Acked 1003 Read 9 Done Done Done false
60962024-01-10T17:57:39.679Z 5 Acked 1004 Write 5 Done Done Sent false
60972024-01-10T17:57:39.679Z 6 Acked 1005 Read 2 Done Done Sent false
60982024-01-10T17:57:39.679Z 7 Acked 1006 Read 7 Done Done Sent false
60992024-01-10T17:57:39.679Z 8 Acked 1007 Read 4 Done Done Sent false
61002024-01-10T17:57:39.679Z 9 Acked 1008 Read 7 Done Done Sent false
61012024-01-10T17:57:39.679Z 10 Acked 1009 Write 9 Done Done Sent false
61022024-01-10T17:57:39.680Z 11 Acked 1010 Read 1 Done Done Sent false
61032024-01-10T17:57:39.680Z 12 Acked 1011 Read 4 Done Done Sent false
61042024-01-10T17:57:39.680Z 13 Acked 1012 Read 10 Done Done Sent false
61052024-01-10T17:57:39.680Z 14 Acked 1013 Read 5 Done Done Sent false
61062024-01-10T17:57:39.680Z 15 Acked 1014 Flush 0 Done Done Sent false
61072024-01-10T17:57:39.680Z 16 Acked 1015 Flush 0 Done Done Sent false
61082024-01-10T17:57:39.681Z 17 Acked 1016 Write 7 Done Done Sent false
61092024-01-10T17:57:39.681Z 18 Acked 1017 Write 4 Done Done Sent false
61102024-01-10T17:57:39.682Z 19 Acked 1018 Write 9 Done Done Sent false
61112024-01-10T17:57:39.682Z 20 Acked 1019 Read 10 Done Done Sent false
61122024-01-10T17:57:39.682Z 21 Acked 1020 Write 5 Done Done Sent false
61132024-01-10T17:57:39.682Z 22 Acked 1021 Flush 0 Done Done Sent false
61142024-01-10T17:57:39.682Z 23 Acked 1022 Write 2 Done Done Sent false
61152024-01-10T17:57:39.682Z 24 Acked 1023 Write 5 Done Done Sent false
61162024-01-10T17:57:39.682Z 25 Acked 1024 Write 3 Done Done Sent false
61172024-01-10T17:57:39.683Z 26 Acked 1025 Write 10 Done Done Sent false
61182024-01-10T17:57:39.683Z 27 Acked 1026 Read 5 Done Done Sent false
61192024-01-10T17:57:39.683Z 28 Acked 1027 Read 8 Done Done Sent false
61202024-01-10T17:57:39.683Z 29 Acked 1028 Write 10 Sent Sent Sent false
61212024-01-10T17:57:39.683Z 30 Acked 1029 Write 5 Sent Sent Sent false
61222024-01-10T17:57:39.683Z STATES DS:0 DS:1 DS:2 TOTAL
61232024-01-10T17:57:39.683Z New 0 0 0 0
61242024-01-10T17:57:39.684Z Sent 2 2 26 30
61252024-01-10T17:57:39.684Z Done 28 28 4 60
61262024-01-10T17:57:39.684Z Skipped 0 0 0 0
61272024-01-10T17:57:39.684Z Error 0 0 0 0
61282024-01-10T17:57:39.684ZLast Flush: 1021 1021 0
61292024-01-10T17:57:39.684ZDownstairs last five completed:
61302024-01-10T17:57:39.684ZUpstairs last five completed: 30 29 28 27 26
61312024-01-10T17:57:39.684ZWrote out file "/tmp/test_repair_verify.data"
61322024-01-10T17:57:39.692Z
61332024-01-10T17:57:39.692ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61342024-01-10T17:57:39.716ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61352024-01-10T17:57:39.716Z 0 000-019 48 48 48 91 91 91 F F F
61362024-01-10T17:57:39.716Z 1 020-039 55 55 55 104 104 104 F F F
61372024-01-10T17:57:39.716Z 2 040-059 56 56 55 105 105 104 F F T <---
61382024-01-10T17:57:39.716Z 3 060-079 56 56 40 105 105 73 F F T <---
61392024-01-10T17:57:39.716Z 4 080-099 52 52 52 99 99 99 F F F
61402024-01-10T17:57:39.716Z 5 100-119 52 52 52 99 99 99 T T F <---
61412024-01-10T17:57:39.716Z 6 120-139 50 50 50 94 94 94 F F F
61422024-01-10T17:57:39.716Z 7 140-159 56 56 44 105 105 84 F F F <---
61432024-01-10T17:57:39.720Z 8 160-179 54 54 54 103 103 103 F F F
61442024-01-10T17:57:39.720Z 9 180-199 54 54 54 103 103 103 F F F
61452024-01-10T17:57:39.720Z 10 200-219 53 53 53 101 101 101 F F F
61462024-01-10T17:57:39.720Z 11 220-239 51 51 51 96 96 96 F F F
61472024-01-10T17:57:39.720Z 12 240-259 56 56 52 105 105 99 F F F <---
61482024-01-10T17:57:39.720Z 13 260-279 56 56 55 107 107 104 F F F <---
61492024-01-10T17:57:39.720Z 14 280-299 55 55 55 104 104 104 F F F
61502024-01-10T17:57:39.720Z 15 300-319 54 54 54 102 102 102 F F F
61512024-01-10T17:57:39.720Z 16 320-339 37 37 37 70 70 70 F F F
61522024-01-10T17:57:39.720Z 17 340-359 53 53 53 101 101 101 F F F
61532024-01-10T17:57:39.720Z 18 360-379 53 53 53 101 101 101 F F F
61542024-01-10T17:57:39.720Z 19 380-399 56 56 55 105 105 104 F F T <---
61552024-01-10T17:57:39.720Z 20 400-419 56 56 55 107 107 104 T T T <---
61562024-01-10T17:57:39.720Z 21 420-439 54 54 54 103 103 103 T T F <---
61572024-01-10T17:57:39.720Z 22 440-459 50 50 50 95 95 95 T T F <---
61582024-01-10T17:57:39.721Z 23 460-479 49 49 49 93 93 93 F F F
61592024-01-10T17:57:39.721Z 24 480-499 56 56 55 107 107 104 T T F <---
61602024-01-10T17:57:39.721Z 25 500-519 55 55 55 104 104 104 F F F
61612024-01-10T17:57:39.721Z 26 520-539 54 54 54 103 103 103 T T F <---
61622024-01-10T17:57:39.721Z 27 540-559 54 54 54 103 103 103 F F F
61632024-01-10T17:57:39.721Z 28 560-579 56 56 53 107 107 101 F F F <---
61642024-01-10T17:57:39.721Z 29 580-599 54 54 54 102 102 102 F F F
61652024-01-10T17:57:39.721ZMax gen: 56, Max flush: 107
61662024-01-10T17:57:39.721ZError: Difference in extent metadata found!
61672024-01-10T17:57:39.724ZA Difference in extent metadata is expected here
61682024-01-10T17:57:39.724ZOn loop 28
61692024-01-10T17:57:39.725Z
61702024-01-10T17:57:39.727ZVerifying data now
61712024-01-10T17:57:46.008ZNo change after verify
61722024-01-10T17:57:46.009ZLoop: 28 Downstairs dump after verify (and repair):
61732024-01-10T17:57:46.031ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61742024-01-10T17:57:46.031Z 0 000-019 48 48 48 91 91 91 F F F
61752024-01-10T17:57:46.031Z 1 020-039 55 55 55 104 104 104 F F F
61762024-01-10T17:57:46.031Z 2 040-059 56 56 56 105 105 105 F F F
61772024-01-10T17:57:46.031Z 3 060-079 56 56 56 105 105 105 F F F
61782024-01-10T17:57:46.031Z 4 080-099 52 52 52 99 99 99 F F F
61792024-01-10T17:57:46.031Z 5 100-119 57 57 57 108 108 108 F F F
61802024-01-10T17:57:46.031Z 6 120-139 50 50 50 94 94 94 F F F
61812024-01-10T17:57:46.031Z 7 140-159 56 56 56 105 105 105 F F F
61822024-01-10T17:57:46.031Z 8 160-179 54 54 54 103 103 103 F F F
61832024-01-10T17:57:46.031Z 9 180-199 54 54 54 103 103 103 F F F
61842024-01-10T17:57:46.031Z 10 200-219 53 53 53 101 101 101 F F F
61852024-01-10T17:57:46.035Z 11 220-239 51 51 51 96 96 96 F F F
61862024-01-10T17:57:46.035Z 12 240-259 56 56 56 105 105 105 F F F
61872024-01-10T17:57:46.035Z 13 260-279 56 56 56 107 107 107 F F F
61882024-01-10T17:57:46.035Z 14 280-299 55 55 55 104 104 104 F F F
61892024-01-10T17:57:46.035Z 15 300-319 54 54 54 102 102 102 F F F
61902024-01-10T17:57:46.035Z 16 320-339 37 37 37 70 70 70 F F F
61912024-01-10T17:57:46.035Z 17 340-359 53 53 53 101 101 101 F F F
61922024-01-10T17:57:46.036Z 18 360-379 53 53 53 101 101 101 F F F
61932024-01-10T17:57:46.036Z 19 380-399 56 56 56 105 105 105 F F F
61942024-01-10T17:57:46.036Z 20 400-419 57 57 57 108 108 108 F F F
61952024-01-10T17:57:46.036Z 21 420-439 57 57 57 108 108 108 F F F
61962024-01-10T17:57:46.036Z 22 440-459 57 57 57 108 108 108 F F F
61972024-01-10T17:57:46.036Z 23 460-479 49 49 49 93 93 93 F F F
61982024-01-10T17:57:46.036Z 24 480-499 57 57 57 108 108 108 F F F
61992024-01-10T17:57:46.036Z 25 500-519 55 55 55 104 104 104 F F F
62002024-01-10T17:57:46.036Z 26 520-539 57 57 57 108 108 108 F F F
62012024-01-10T17:57:46.036Z 27 540-559 54 54 54 103 103 103 F F F
62022024-01-10T17:57:46.036Z 28 560-579 56 56 56 107 107 107 F F F
62032024-01-10T17:57:46.036Z 29 580-599 54 54 54 102 102 102 F F F
62042024-01-10T17:57:46.036ZMax gen: 57, Max flush: 108
62052024-01-10T17:57:46.039Z
62062024-01-10T17:57:46.039ZBegin loop 29
62072024-01-10T17:57:46.040ZDownstairs to restart: 1
62082024-01-10T17:57:46.071Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.069598589Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387}
62092024-01-10T17:57:46.072Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.07005059Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387}
62102024-01-10T17:57:46.072Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070094027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387}
62112024-01-10T17:57:46.083Z{"msg":"Crucible 078abd99-f747-4cde-81ce-7ffa162ccf40 has session id: 0112868e-690c-4061-960f-02769801e9c0","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070165262Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62122024-01-10T17:57:46.083Z{"msg":"Upstairs opts: Upstairs UUID: 078abd99-f747-4cde-81ce-7ffa162ccf40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070252583Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62132024-01-10T17:57:46.083Z{"msg":"Crucible stats registered with UUID: 078abd99-f747-4cde-81ce-7ffa162ccf40","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070325795Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62142024-01-10T17:57:46.083ZCrucible runtime is spawned
62152024-01-10T17:57:46.083Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070583851Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387}
62162024-01-10T17:57:46.083Z{"msg":"078abd99-f747-4cde-81ce-7ffa162ccf40 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070642779Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62172024-01-10T17:57:46.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070671337Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"0","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62182024-01-10T17:57:46.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070778938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"0","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62192024-01-10T17:57:46.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070823068Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"1","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62202024-01-10T17:57:46.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070896675Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"1","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62212024-01-10T17:57:46.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.070942141Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"2","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62222024-01-10T17:57:46.085Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.071013513Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"2","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62232024-01-10T17:57:46.085Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.07111941Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62242024-01-10T17:57:46.085Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.071354244Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62252024-01-10T17:57:46.085Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.07144418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62262024-01-10T17:57:46.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.071517743Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62272024-01-10T17:57:46.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.071561874Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62282024-01-10T17:57:46.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.071631855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62292024-01-10T17:57:46.085Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.072519982Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"2","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62302024-01-10T17:57:46.085Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.07260066Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"2","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62312024-01-10T17:57:46.085Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.072629916Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"0","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62322024-01-10T17:57:46.087Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.072671717Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"0","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62332024-01-10T17:57:46.087Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.073503547Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"2","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62342024-01-10T17:57:46.087Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.073760617Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"0","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62352024-01-10T17:57:46.087Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.074338295Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"2","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62362024-01-10T17:57:46.087Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.074428355Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"2","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62372024-01-10T17:57:46.087Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.074477027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62382024-01-10T17:57:46.087Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.074611132Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"1","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62392024-01-10T17:57:46.087Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.074691718Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"1","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62402024-01-10T17:57:46.087Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.074768592Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"0","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62412024-01-10T17:57:46.087Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.074839524Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"0","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62422024-01-10T17:57:46.089Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.074883149Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62432024-01-10T17:57:46.089Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.075684456Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"1","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62442024-01-10T17:57:46.089Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076373786Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"1","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62452024-01-10T17:57:46.089Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076431691Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"1","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62462024-01-10T17:57:46.089Z{"msg":"[0]R flush_numbers[0..12]: [91, 104, 105, 105, 99, 108, 94, 105, 103, 103, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076458008Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62472024-01-10T17:57:46.089Z{"msg":"[0]R generation[0..12]: [48, 55, 56, 56, 52, 57, 50, 56, 54, 54, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076499Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62482024-01-10T17:57:46.089Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076533026Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62492024-01-10T17:57:46.089Z{"msg":"[1]R flush_numbers[0..12]: [91, 104, 105, 105, 99, 108, 94, 105, 103, 103, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076576982Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62502024-01-10T17:57:46.089Z{"msg":"[1]R generation[0..12]: [48, 55, 56, 56, 52, 57, 50, 56, 54, 54, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076601656Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62512024-01-10T17:57:46.089Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076647843Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62522024-01-10T17:57:46.089Z{"msg":"[2]R flush_numbers[0..12]: [91, 104, 105, 105, 99, 108, 94, 105, 103, 103, 101, 96]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076672537Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62532024-01-10T17:57:46.089Z{"msg":"[2]R generation[0..12]: [48, 55, 56, 56, 52, 57, 50, 56, 54, 54, 53, 51]","v":0Wait for a query_work_queue command to finish before sending IO
62542024-01-10T17:57:46.089Z,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076701309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62552024-01-10T17:57:46.090Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076725733Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62562024-01-10T17:57:46.090Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.07674711Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62572024-01-10T17:57:46.091Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076770435Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62582024-01-10T17:57:46.091Z{"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076791419Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62592024-01-10T17:57:46.091Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076844898Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"":"downstairs","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62602024-01-10T17:57:46.091Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076905881Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62612024-01-10T17:57:46.091Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076928715Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"0","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62622024-01-10T17:57:46.091Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.076963044Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"1","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62632024-01-10T17:57:46.091Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.077027913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"client":"2","":"downstairs","session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62642024-01-10T17:57:46.091Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.077054195Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62652024-01-10T17:57:46.091Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.07707457Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62662024-01-10T17:57:46.091Z{"msg":"078abd99-f747-4cde-81ce-7ffa162ccf40 is now active with session: 0112868e-690c-4061-960f-02769801e9c0","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.077116882Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62672024-01-10T17:57:46.091Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.077137417Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387,"session_id":"0112868e-690c-4061-960f-02769801e9c0"}
62682024-01-10T17:57:46.091Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:46.077164346Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1387}
62692024-01-10T17:57:46.091ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
62702024-01-10T17:57:46.091ZLoading write count information from file "/tmp/test_repair_verify.data"
62712024-01-10T17:57:46.091ZRun Repair workload
62722024-01-10T17:57:46.091Z01/30 Write block 217 len 36864 data: 8 5 5 5 5 5 4 3 4
62732024-01-10T17:57:46.091Z02/30 Read block 474 len 20480
62742024-01-10T17:57:46.101Z03/30 Read block 353 len 8192
62752024-01-10T17:57:46.101Z04/30 Write block 185 len 36864 data: 7 7 6 7 7 7 7 7 6
62762024-01-10T17:57:46.101Z05/30 Read block 239 len 40960
62772024-01-10T17:57:46.101Z06/30 Write block 79 len 36864 data: 6 5 6 5 6 4 3 3 3
62782024-01-10T17:57:46.101Z07/30 Read block 547 len 16384
62792024-01-10T17:57:46.101Z08/30 Write block 491 len 24576 data: 8 9 10 10 13 11
62802024-01-10T17:57:46.101Z09/30 Read block 176 len 8192
62812024-01-10T17:57:46.104Z10/30 Read block 16 len 8192
62822024-01-10T17:57:46.104Z11/30 Flush
62832024-01-10T17:57:46.105Z12/30 Flush
62842024-01-10T17:57:46.105Z13/30 Write block 362 len 36864 data: 4 4 4 3 3 3 4 4 6
62852024-01-10T17:57:46.105Z14/30 Write block 325 len 28672 data: 7 5 5 6 5 5 4
62862024-01-10T17:57:46.105Z15/30 Read block 576 len 12288
62872024-01-10T17:57:46.108Z16/30 Read block 404 len 32768
62882024-01-10T17:57:46.111Z17/30 Read block 593 len 24576
62892024-01-10T17:57:46.114Z18/30 Flush
62902024-01-10T17:57:46.118Z19/30 Write block 74 len 12288 data: 3 3 3
62912024-01-10T17:57:46.118Z20/30 Read block 116 len 32768
62922024-01-10T17:57:46.127Z21/30 Write block 230 len 4096 data: 3
62932024-01-10T17:57:46.127Z22/30 Write block 387 len 24576 data: 3 2 2 2 3 3
62942024-01-10T17:57:46.127Z23/30 Write block 48 len 20480 data: 4 6 5 4 4
62952024-01-10T17:57:46.127Z24/30 Flush
62962024-01-10T17:57:46.135Z25/30 Write block 223 len 36864 data: 5 4 5 4 4 4 4 4 2
62972024-01-10T17:57:46.136Z26/30 Write block 225 len 20480 data: 6 5 5 5 5
62982024-01-10T17:57:46.136Z27/30 Write block 187 len 40960 data: 7 8 8 8 8 8 7 8 8 5
62992024-01-10T17:57:46.136Z28/30 Read block 366 len 24576
63002024-01-10T17:57:46.136Z29/30 Write block 423 len 8192 data: 8 7
63012024-01-10T17:57:46.148Z30/30 Read block 166 len 12288
63022024-01-10T17:57:46.148Z----------------------------------------------------------------
63032024-01-10T17:57:46.148Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
63042024-01-10T17:57:46.148ZStates: Active Active Active
63052024-01-10T17:57:46.148ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63062024-01-10T17:57:46.148Z 1 Acked 1000 Write 9 Done Done Done false
63072024-01-10T17:57:46.148Z 2 Acked 1001 Read 5 Done Done Done false
63082024-01-10T17:57:46.148Z 3 Acked 1002 Read 2 Done Sent Done false
63092024-01-10T17:57:46.148Z 4 Acked 1003 Write 9 Done Sent Done false
63102024-01-10T17:57:46.148Z 5 Acked 1004 Read 10 Done Sent Done false
63112024-01-10T17:57:46.148Z 6 Acked 1005 Write 9 Done Sent Done false
63122024-01-10T17:57:46.148Z 7 Acked 1006 Read 4 Done Sent Done false
63132024-01-10T17:57:46.148Z 8 Acked 1007 Write 6 Done Sent Done false
63142024-01-10T17:57:46.148Z 9 Acked 1008 Read 2 Done Sent Done false
63152024-01-10T17:57:46.148Z 10 Acked 1009 Read 2 Done Sent Done false
63162024-01-10T17:57:46.148Z 11 Acked 1010 Flush 0 Done Sent Done false
63172024-01-10T17:57:46.148Z 12 Acked 1011 Flush 0 Done Sent Done false
63182024-01-10T17:57:46.148Z 13 Acked 1012 Write 9 Done Sent Done false
63192024-01-10T17:57:46.148Z 14 Acked 1013 Write 7 Done Sent Done false
63202024-01-10T17:57:46.150Z 15 Acked 1014 Read 3 Done Sent Done false
63212024-01-10T17:57:46.150Z 16 Acked 1015 Read 8 Done Sent Done false
63222024-01-10T17:57:46.150Z 17 Acked 1016 Read 6 Done Sent Done false
63232024-01-10T17:57:46.150Z 18 Acked 1017 Flush 0 Done Sent Done false
63242024-01-10T17:57:46.150Z 19 Acked 1018 Write 3 Done Sent Done false
63252024-01-10T17:57:46.150Z 20 Acked 1019 Read 8 Done Sent Done false
63262024-01-10T17:57:46.150Z 21 Acked 1020 Write 1 Done Sent Done false
63272024-01-10T17:57:46.150Z 22 Acked 1021 Write 6 Done Sent Done false
63282024-01-10T17:57:46.150Z 23 Acked 1022 Write 5 Done Sent Done false
63292024-01-10T17:57:46.150Z 24 Acked 1023 Flush 0 Done Sent Done false
63302024-01-10T17:57:46.150Z 25 Acked 1024 Write 9 Done Sent Done false
63312024-01-10T17:57:46.150Z 26 Acked 1025 Write 5 Done Sent Done false
63322024-01-10T17:57:46.150Z 27 Acked 1026 Write 10 Done Sent Done false
63332024-01-10T17:57:46.150Z 28 Acked 1027 Read 6 Done Sent Done false
63342024-01-10T17:57:46.150Z 29 Acked 1028 Write 2 Done Sent Done false
63352024-01-10T17:57:46.150Z 30 Acked 1029 Read 3 Done Sent Sent false
63362024-01-10T17:57:46.150Z STATES DS:0 DS:1 DS:2 TOTAL
63372024-01-10T17:57:46.150Z New 0 0 0 0
63382024-01-10T17:57:46.150Z Sent 0 28 1 29
63392024-01-10T17:57:46.150Z Done 30 2 29 61
63402024-01-10T17:57:46.150Z Skipped 0 0 0 0
63412024-01-10T17:57:46.151Z Error 0 0 0 0
63422024-01-10T17:57:46.151ZLast Flush: 1023 0 1023
63432024-01-10T17:57:46.151ZDownstairs last five completed:
63442024-01-10T17:57:46.151ZUpstairs last five completed: 30 29 28 27 26
63452024-01-10T17:57:46.151ZWrote out file "/tmp/test_repair_verify.data"
63462024-01-10T17:57:46.158Z
63472024-01-10T17:57:46.158ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63482024-01-10T17:57:46.177ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63492024-01-10T17:57:46.177Z 0 000-019 48 48 48 91 91 91 F F F
63502024-01-10T17:57:46.177Z 1 020-039 55 55 55 104 104 104 F F F
63512024-01-10T17:57:46.177Z 2 040-059 58 56 58 112 105 112 F F F <---
63522024-01-10T17:57:46.177Z 3 060-079 58 56 58 112 105 112 F F F <---
63532024-01-10T17:57:46.177Z 4 080-099 58 52 58 109 99 109 F F F <---
63542024-01-10T17:57:46.186Z 5 100-119 57 57 57 108 108 108 F F F
63552024-01-10T17:57:46.186Z 6 120-139 50 50 50 94 94 94 F F F
63562024-01-10T17:57:46.186Z 7 140-159 56 56 56 105 105 105 F F F
63572024-01-10T17:57:46.186Z 8 160-179 54 54 54 103 103 103 F F F
63582024-01-10T17:57:46.186Z 9 180-199 58 54 58 109 103 109 T F T <---
63592024-01-10T17:57:46.186Z 10 200-219 58 53 58 109 101 109 F T F <---
63602024-01-10T17:57:46.186Z 11 220-239 58 51 58 112 96 112 T T T <---
63612024-01-10T17:57:46.186Z 12 240-259 56 56 56 105 105 105 F F F
63622024-01-10T17:57:46.186Z 13 260-279 56 56 56 107 107 107 F F F
63632024-01-10T17:57:46.186Z 14 280-299 55 55 55 104 104 104 F F F
63642024-01-10T17:57:46.186Z 15 300-319 54 54 54 102 102 102 F F F
63652024-01-10T17:57:46.186Z 16 320-339 58 37 58 111 70 111 F F F <---
63662024-01-10T17:57:46.186Z 17 340-359 53 53 53 101 101 101 F F F
63672024-01-10T17:57:46.186Z 18 360-379 58 53 58 111 101 111 F F F <---
63682024-01-10T17:57:46.186Z 19 380-399 58 56 58 112 105 112 F F F <---
63692024-01-10T17:57:46.186Z 20 400-419 57 57 57 108 108 108 F F F
63702024-01-10T17:57:46.186Z 21 420-439 57 57 57 108 108 108 T F T <---
63712024-01-10T17:57:46.186Z 22 440-459 57 57 57 108 108 108 F F F
63722024-01-10T17:57:46.186Z 23 460-479 49 49 49 93 93 93 F F F
63732024-01-10T17:57:46.186Z 24 480-499 58 57 58 109 108 109 F F F <---
63742024-01-10T17:57:46.187Z 25 500-519 55 55 55 104 104 104 F F F
63752024-01-10T17:57:46.187Z 26 520-539 57 57 57 108 108 108 F F F
63762024-01-10T17:57:46.187Z 27 540-559 54 54 54 103 103 103 F F F
63772024-01-10T17:57:46.187Z 28 560-579 56 56 56 107 107 107 F F F
63782024-01-10T17:57:46.187Z 29 580-599 54 54 54 102 102 102 F F F
63792024-01-10T17:57:46.187ZMax gen: 58, Max flush: 112
63802024-01-10T17:57:46.187ZError: Difference in extent metadata found!
63812024-01-10T17:57:46.187ZA Difference in extent metadata is expected here
63822024-01-10T17:57:46.187ZOn loop 29
63832024-01-10T17:57:46.187Z
63842024-01-10T17:57:46.187ZVerifying data now
63852024-01-10T17:57:51.306ZNo change after verify
63862024-01-10T17:57:51.307ZLoop: 29 Downstairs dump after verify (and repair):
63872024-01-10T17:57:51.332ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63882024-01-10T17:57:51.332Z 0 000-019 48 48 48 91 91 91 F F F
63892024-01-10T17:57:51.333Z 1 020-039 55 55 55 104 104 104 F F F
63902024-01-10T17:57:51.333Z 2 040-059 58 58 58 112 112 112 F F F
63912024-01-10T17:57:51.333Z 3 060-079 58 58 58 112 112 112 F F F
63922024-01-10T17:57:51.334Z 4 080-099 58 58 58 109 109 109 F F F
63932024-01-10T17:57:51.334Z 5 100-119 57 57 57 108 108 108 F F F
63942024-01-10T17:57:51.334Z 6 120-139 50 50 50 94 94 94 F F F
63952024-01-10T17:57:51.354Z 7 140-159 56 56 56 105 105 105 F F F
63962024-01-10T17:57:51.355Z 8 160-179 54 54 54 103 103 103 F F F
63972024-01-10T17:57:51.355Z 9 180-199 59 59 59 113 113 113 F F F
63982024-01-10T17:57:51.355Z 10 200-219 58 58 58 109 109 109 F F F
63992024-01-10T17:57:51.355Z 11 220-239 59 59 59 113 113 113 F F F
64002024-01-10T17:57:51.356Z 12 240-259 56 56 56 105 105 105 F F F
64012024-01-10T17:57:51.356Z 13 260-279 56 56 56 107 107 107 F F F
64022024-01-10T17:57:51.357Z 14 280-299 55 55 55 104 104 104 F F F
64032024-01-10T17:57:51.357Z 15 300-319 54 54 54 102 102 102 F F F
64042024-01-10T17:57:51.358Z 16 320-339 58 58 58 111 111 111 F F F
64052024-01-10T17:57:51.358Z 17 340-359 53 53 53 101 101 101 F F F
64062024-01-10T17:57:51.358Z 18 360-379 58 58 58 111 111 111 F F F
64072024-01-10T17:57:51.358Z 19 380-399 58 58 58 112 112 112 F F F
64082024-01-10T17:57:51.359Z 20 400-419 57 57 57 108 108 108 F F F
64092024-01-10T17:57:51.359Z 21 420-439 59 59 59 113 113 113 F F F
64102024-01-10T17:57:51.359Z 22 440-459 57 57 57 108 108 108 F F F
64112024-01-10T17:57:51.359Z 23 460-479 49 49 49 93 93 93 F F F
64122024-01-10T17:57:51.359Z 24 480-499 58 58 58 109 109 109 F F F
64132024-01-10T17:57:51.359Z 25 500-519 55 55 55 104 104 104 F F F
64142024-01-10T17:57:51.359Z 26 520-539 57 57 57 108 108 108 F F F
64152024-01-10T17:57:51.360Z 27 540-559 54 54 54 103 103 103 F F F
64162024-01-10T17:57:51.360Z 28 560-579 56 56 56 107 107 107 F F F
64172024-01-10T17:57:51.360Z 29 580-599 54 54 54 102 102 102 F F F
64182024-01-10T17:57:51.360ZMax gen: 59, Max flush: 113
64192024-01-10T17:57:51.361Z
64202024-01-10T17:57:51.361ZBegin loop 30
64212024-01-10T17:57:51.361ZDownstairs to restart: 1
64222024-01-10T17:57:51.373Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.356232305Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395}
64232024-01-10T17:57:51.373Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.356782956Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395}
64242024-01-10T17:57:51.373Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.356816519Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395}
64252024-01-10T17:57:51.373Z{"msg":"Crucible 53443c4c-8c3c-4be5-86f7-398da425ef98 has session id: 8e530dd4-d80e-406f-ab5d-d1c5c07558b2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.356878095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64262024-01-10T17:57:51.373Z{"msg":"Upstairs opts: Upstairs UUID: 53443c4c-8c3c-4be5-86f7-398da425ef98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.356921701Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64272024-01-10T17:57:51.373Z{"msg":"Crucible stats registered with UUID: 53443c4c-8c3c-4be5-86f7-398da425ef98","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.356961481Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64282024-01-10T17:57:51.373ZCrucible runtime is spawned
64292024-01-10T17:57:51.373Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.357208177Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395}
64302024-01-10T17:57:51.374Z{"msg":"53443c4c-8c3c-4be5-86f7-398da425ef98 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.35745398Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64312024-01-10T17:57:51.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.357529019Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"0","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64322024-01-10T17:57:51.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.357626048Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"0","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64332024-01-10T17:57:51.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.35770386Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"1","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64342024-01-10T17:57:51.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.357748373Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"1","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64352024-01-10T17:57:51.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.357788158Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"2","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64362024-01-10T17:57:51.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.357873836Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"2","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64372024-01-10T17:57:51.374Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.357945953Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64382024-01-10T17:57:51.374Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.358019035Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64392024-01-10T17:57:51.374Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.358062846Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64402024-01-10T17:57:51.374Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.358136107Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64412024-01-10T17:57:51.376Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.358169768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64422024-01-10T17:57:51.376Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.358228877Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64432024-01-10T17:57:51.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.359513929Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"0","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64442024-01-10T17:57:51.377Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.359621641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"0","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64452024-01-10T17:57:51.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.359672738Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"2","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64462024-01-10T17:57:51.377Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.359715625Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"2","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64472024-01-10T17:57:51.377Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.360518951Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"0","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64482024-01-10T17:57:51.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.360757461Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"1","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64492024-01-10T17:57:51.377Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.360815328Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"1","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64502024-01-10T17:57:51.377Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.360841768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"2","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64512024-01-10T17:57:51.378Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.361239142Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"0","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64522024-01-10T17:57:51.378Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.361297971Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"0","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64532024-01-10T17:57:51.378Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.361433187Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64542024-01-10T17:57:51.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.361634908Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"2","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64552024-01-10T17:57:51.379Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.361731994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"2","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64562024-01-10T17:57:51.379Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.361809276Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64572024-01-10T17:57:51.379Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.362036927Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"1","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64582024-01-10T17:57:51.380Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.362725567Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"1","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64592024-01-10T17:57:51.380Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.362812899Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"1","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64602024-01-10T17:57:51.380Z{"msg":"[0]R flush_numbers[0..12]: [91, 104, 112, 112, 109, 108, 94, 105, 103, 113, 109, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.362888865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64612024-01-10T17:57:51.380Z{"msg":"[0]R generation[0..12]: [48, 55, 58, 58, 58, 57, 50, 56, 54, 59, 58, 59]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.362960272Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64622024-01-10T17:57:51.380Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363035818Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64632024-01-10T17:57:51.380Z{"msg":"[1]R flush_numbers[0..12]: [91, 104, 112, 112, 109, 108, 94, 105, 103, 113, 109, 113]Wait for a query_work_queue command to finish before sending IO
64642024-01-10T17:57:51.380Z","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363199917Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64652024-01-10T17:57:51.380Z{"msg":"[1]R generation[0..12]: [48, 55, 58, 58, 58, 57, 50, 56, 54, 59, 58, 59]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363276747Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64662024-01-10T17:57:51.380Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363353635Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64672024-01-10T17:57:51.380Z{"msg":"[2]R flush_numbers[0..12]: [91, 104, 112, 112, 109, 108, 94, 105, 103, 113, 109, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363425956Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64682024-01-10T17:57:51.380Z{"msg":"[2]R generation[0..12]: [48, 55, 58, 58, 58, 57, 50, 56, 54, 59, 58, 59]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.3635035Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64692024-01-10T17:57:51.380Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363582745Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64702024-01-10T17:57:51.380Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363658259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64712024-01-10T17:57:51.381Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
64722024-01-10T17:57:51.381Z2024-01-10T17:57:51.363734632Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64732024-01-10T17:57:51.381Z{"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363855911Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64742024-01-10T17:57:51.381Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363905391Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"":"downstairs","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64752024-01-10T17:57:51.382Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363928787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64762024-01-10T17:57:51.382Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363966512Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"0","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64772024-01-10T17:57:51.382Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.363989453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"1","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64782024-01-10T17:57:51.382Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.364028712Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"client":"2","":"downstairs","session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64792024-01-10T17:57:51.382Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.364051439Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64802024-01-10T17:57:51.383Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.364089092Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64812024-01-10T17:57:51.383Z{"msg":"53443c4c-8c3c-4be5-86f7-398da425ef98 is now active with session: 8e530dd4-d80e-406f-ab5d-d1c5c07558b2","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.364111513Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64822024-01-10T17:57:51.383Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.364130619Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395,"session_id":"8e530dd4-d80e-406f-ab5d-d1c5c07558b2"}
64832024-01-10T17:57:51.383Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:51.364150404Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1395}
64842024-01-10T17:57:51.383ZLoading write count information from file "/tmp/test_repair_verify.data"
64852024-01-10T17:57:51.383ZRun Repair workload
64862024-01-10T17:57:51.383Z01/30 Write block 405 len 24576 data: 7 8 8 6 5 7
64872024-01-10T17:57:51.384Z02/30 Read block 527 len 24576
64882024-01-10T17:57:51.384Z03/30 Read block 180 len 20480
64892024-01-10T17:57:51.399Z04/30 Write block 456 len 24576 data: 3 3 3 2 3 3
64902024-01-10T17:57:51.399Z05/30 Flush
64912024-01-10T17:57:51.399Z06/30 Flush
64922024-01-10T17:57:51.399Z07/30 Write block 311 len 4096 data: 4
64932024-01-10T17:57:51.399Z08/30 Write block 514 len 36864 data: 9 7 7 7 6 5 5 5 3
64942024-01-10T17:57:51.399Z09/30 Write block 590 len 36864 data: 6 6 5 6 7 6 5 4 3
64952024-01-10T17:57:51.399Z10/30 Flush
64962024-01-10T17:57:51.400Z11/30 Write block 127 len 16384 data: 3 2 2 5
64972024-01-10T17:57:51.400Z12/30 Read block 205 len 16384
64982024-01-10T17:57:51.400Z13/30 Write block 227 len 20480 data: 6 6 6 5 3
64992024-01-10T17:57:51.400Z14/30 Write block 477 len 4096 data: 4
65002024-01-10T17:57:51.400Z15/30 Write block 509 len 28672 data: 8 8 8 9 9 10 8
65012024-01-10T17:57:51.400Z16/30 Write block 209 len 32768 data: 5 6 7 7 7 7 6 9
65022024-01-10T17:57:51.400Z17/30 Read block 400 len 32768
65032024-01-10T17:57:51.400Z18/30 Flush
65042024-01-10T17:57:51.400Z19/30 Flush
65052024-01-10T17:57:51.400Z20/30 Write block 244 len 28672 data: 8 7 5 4 4 5 4
65062024-01-10T17:57:51.400Z21/30 Read block 534 len 32768
65072024-01-10T17:57:51.411Z22/30 Write block 476 len 4096 data: 4
65082024-01-10T17:57:51.411Z23/30 Read block 558 len 16384
65092024-01-10T17:57:51.411Z24/30 Write block 110 len 32768 data: 5 6 7 7 6 5 5 5
65102024-01-10T17:57:51.411Z25/30 Flush
65112024-01-10T17:57:51.412Z26/30 Write block 160 len 32768 data: 5 7 10 10 10 10 10 9
65122024-01-10T17:57:51.412Z27/30 Read block 171 len 24576
65132024-01-10T17:57:51.425Z28/30 Write block 243 len 4096 data: 5
65142024-01-10T17:57:51.425Z29/30 Read block 483 len 28672
65152024-01-10T17:57:51.425Z30/30 Read block 194 len 28672
65162024-01-10T17:57:51.425Z----------------------------------------------------------------
65172024-01-10T17:57:51.425Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65182024-01-10T17:57:51.425ZStates: Active Active Active
65192024-01-10T17:57:51.425ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65202024-01-10T17:57:51.425Z 1 Acked 1000 Write 6 Done Done Done false
65212024-01-10T17:57:51.425Z 2 Acked 1001 Read 6 Done Done Done false
65222024-01-10T17:57:51.425Z 3 Acked 1002 Read 5 Done Sent Done false
65232024-01-10T17:57:51.425Z 4 Acked 1003 Write 6 Done Sent Done false
65242024-01-10T17:57:51.425Z 5 Acked 1004 Flush 0 Done Sent Done false
65252024-01-10T17:57:51.426Z 6 Acked 1005 Flush 0 Done Sent Done false
65262024-01-10T17:57:51.426Z 7 Acked 1006 Write 1 Done Sent Done false
65272024-01-10T17:57:51.426Z 8 Acked 1007 Write 9 Done Sent Done false
65282024-01-10T17:57:51.426Z 9 Acked 1008 Write 9 Done Sent Done false
65292024-01-10T17:57:51.426Z 10 Acked 1009 Flush 0 Done Sent Done false
65302024-01-10T17:57:51.426Z 11 Acked 1010 Write 4 Done Sent Done false
65312024-01-10T17:57:51.426Z 12 Acked 1011 Read 4 Done Sent Done false
65322024-01-10T17:57:51.426Z 13 Acked 1012 Write 5 Done Sent Done false
65332024-01-10T17:57:51.426Z 14 Acked 1013 Write 1 Done Sent Done false
65342024-01-10T17:57:51.428Z 15 Acked 1014 Write 7 Done Sent Done false
65352024-01-10T17:57:51.428Z 16 Acked 1015 Write 8 Done Sent Done false
65362024-01-10T17:57:51.428Z 17 Acked 1016 Read 8 Done Sent Done false
65372024-01-10T17:57:51.428Z 18 Acked 1017 Flush 0 Done Sent Done false
65382024-01-10T17:57:51.428Z 19 Acked 1018 Flush 0 Done Sent Done false
65392024-01-10T17:57:51.428Z 20 Acked 1019 Write 7 Done Sent Done false
65402024-01-10T17:57:51.428Z 21 Acked 1020 Read 8 Done Sent Done false
65412024-01-10T17:57:51.428Z 22 Acked 1021 Write 1 Done Sent Done false
65422024-01-10T17:57:51.428Z 23 Acked 1022 Read 4 Done Sent Done false
65432024-01-10T17:57:51.428Z 24 Acked 1023 Write 8 Done Sent Done false
65442024-01-10T17:57:51.430Z 25 Acked 1024 Flush 0 Done Sent Done false
65452024-01-10T17:57:51.430Z 26 Acked 1025 Write 8 Done Sent Done false
65462024-01-10T17:57:51.430Z 27 Acked 1026 Read 6 Done Sent Done false
65472024-01-10T17:57:51.430Z 28 Acked 1027 Write 1 Done Sent Done false
65482024-01-10T17:57:51.430Z 29 Acked 1028 Read 7 Done Sent Done false
65492024-01-10T17:57:51.430Z 30 Acked 1029 Read 7 Done Sent Done false
65502024-01-10T17:57:51.430Z STATES DS:0 DS:1 DS:2 TOTAL
65512024-01-10T17:57:51.430Z New 0 0 0 0
65522024-01-10T17:57:51.430Z Sent 0 28 0 28
65532024-01-10T17:57:51.430Z Done 30 2 30 62
65542024-01-10T17:57:51.430Z Skipped 0 0 0 0
65552024-01-10T17:57:51.430Z Error 0 0 0 0
65562024-01-10T17:57:51.430ZLast Flush: 1024 0 1024
65572024-01-10T17:57:51.430ZDownstairs last five completed:
65582024-01-10T17:57:51.431ZUpstairs last five completed: 30 29 28 27 26
65592024-01-10T17:57:51.431ZWrote out file "/tmp/test_repair_verify.data"
65602024-01-10T17:57:51.513Z
65612024-01-10T17:57:51.513ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
65622024-01-10T17:57:51.524ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65632024-01-10T17:57:51.525Z 0 000-019 48 48 48 91 91 91 F F F
65642024-01-10T17:57:51.525Z 1 020-039 55 55 55 104 104 104 F F F
65652024-01-10T17:57:51.525Z 2 040-059 58 58 58 112 112 112 F F F
65662024-01-10T17:57:51.525Z 3 060-079 58 58 58 112 112 112 F F F
65672024-01-10T17:57:51.525Z 4 080-099 58 58 58 109 109 109 F F F
65682024-01-10T17:57:51.525Z 5 100-119 60 57 60 119 108 119 F F F <---
65692024-01-10T17:57:51.525Z 6 120-139 60 50 60 117 94 117 F F F <---
65702024-01-10T17:57:51.525Z 7 140-159 56 56 56 105 105 105 F F F
65712024-01-10T17:57:51.525Z 8 160-179 54 54 54 103 103 103 T F T <---
65722024-01-10T17:57:51.525Z 9 180-199 59 59 59 113 113 113 F F F
65732024-01-10T17:57:51.525Z 10 200-219 60 58 60 117 109 117 F F F <---
65742024-01-10T17:57:51.525Z 11 220-239 60 59 60 117 113 117 F F F <---
65752024-01-10T17:57:51.526Z 12 240-259 60 56 60 119 105 119 T F T <---
65762024-01-10T17:57:51.526Z 13 260-279 56 56 56 107 107 107 F F F
65772024-01-10T17:57:51.526Z 14 280-299 55 55 55 104 104 104 F F F
65782024-01-10T17:57:51.526Z 15 300-319 60 54 60 116 102 116 F F F <---
65792024-01-10T17:57:51.527Z 16 320-339 58 58 58 111 111 111 F F F
65802024-01-10T17:57:51.527Z 17 340-359 53 53 53 101 101 101 F F F
65812024-01-10T17:57:51.527Z 18 360-379 58 58 58 111 111 111 F F F
65822024-01-10T17:57:51.527Z 19 380-399 58 58 58 112 112 112 F F F
65832024-01-10T17:57:51.527Z 20 400-419 60 57 60 114 108 114 F T F <---
65842024-01-10T17:57:51.527Z 21 420-439 59 59 59 113 113 113 F F F
65852024-01-10T17:57:51.527Z 22 440-459 60 57 60 114 108 114 F F F <---
65862024-01-10T17:57:51.527Z 23 460-479 60 49 60 119 93 119 F F F <---
65872024-01-10T17:57:51.527Z 24 480-499 58 58 58 109 109 109 F F F
65882024-01-10T17:57:51.527Z 25 500-519 60 55 60 117 104 117 F F F <---
65892024-01-10T17:57:51.528Z 26 520-539 60 57 60 116 108 116 F F F <---
65902024-01-10T17:57:51.528Z 27 540-559 54 54 54 103 103 103 F F F
65912024-01-10T17:57:51.528Z 28 560-579 56 56 56 107 107 107 F F F
65922024-01-10T17:57:51.528Z 29 580-599 60 54 60 116 102 116 F F F <---
65932024-01-10T17:57:51.528ZMax gen: 60, Max flush: 119
65942024-01-10T17:57:51.528ZError: Difference in extent metadata found!
65952024-01-10T17:57:51.528ZA Difference in extent metadata is expected here
65962024-01-10T17:57:51.528ZOn loop 30
65972024-01-10T17:57:51.528Z
65982024-01-10T17:57:51.528ZVerifying data now
65992024-01-10T17:58:01.332ZNo change after verify
66002024-01-10T17:58:01.332ZLoop: 30 Downstairs dump after verify (and repair):
66012024-01-10T17:58:01.416ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66022024-01-10T17:58:01.416Z 0 000-019 48 48 48 91 91 91 F F F
66032024-01-10T17:58:01.416Z 1 020-039 55 55 55 104 104 104 F F F
66042024-01-10T17:58:01.416Z 2 040-059 58 58 58 112 112 112 F F F
66052024-01-10T17:58:01.416Z 3 060-079 58 58 58 112 112 112 F F F
66062024-01-10T17:58:01.416Z 4 080-099 58 58 58 109 109 109 F F F
66072024-01-10T17:58:01.416Z 5 100-119 60 60 60 119 119 119 F F F
66082024-01-10T17:58:01.417Z 6 120-139 60 60 60 117 117 117 F F F
66092024-01-10T17:58:01.417Z 7 140-159 56 56 56 105 105 105 F F F
66102024-01-10T17:58:01.417Z 8 160-179 61 61 61 120 120 120 F F F
66112024-01-10T17:58:01.417Z 9 180-199 59 59 59 113 113 113 F F F
66122024-01-10T17:58:01.417Z 10 200-219 60 60 60 117 117 117 F F F
66132024-01-10T17:58:01.417Z 11 220-239 60 60 60 117 117 117 F F F
66142024-01-10T17:58:01.417Z 12 240-259 61 61 61 120 120 120 F F F
66152024-01-10T17:58:01.417Z 13 260-279 56 56 56 107 107 107 F F F
66162024-01-10T17:58:01.418Z 14 280-299 55 55 55 104 104 104 F F F
66172024-01-10T17:58:01.418Z 15 300-319 60 60 60 116 116 116 F F F
66182024-01-10T17:58:01.418Z 16 320-339 58 58 58 111 111 111 F F F
66192024-01-10T17:58:01.418Z 17 340-359 53 53 53 101 101 101 F F F
66202024-01-10T17:58:01.418Z 18 360-379 58 58 58 111 111 111 F F F
66212024-01-10T17:58:01.418Z 19 380-399 58 58 58 112 112 112 F F F
66222024-01-10T17:58:01.418Z 20 400-419 60 60 60 114 114 114 F F F
66232024-01-10T17:58:01.418Z 21 420-439 59 59 59 113 113 113 F F F
66242024-01-10T17:58:01.418Z 22 440-459 60 60 60 114 114 114 F F F
66252024-01-10T17:58:01.418Z 23 460-479 60 60 60 119 119 119 F F F
66262024-01-10T17:58:01.418Z 24 480-499 58 58 58 109 109 109 F F F
66272024-01-10T17:58:01.419Z 25 500-519 60 60 60 117 117 117 F F F
66282024-01-10T17:58:01.419Z 26 520-539 60 60 60 116 116 116 F F F
66292024-01-10T17:58:01.419Z 27 540-559 54 54 54 103 103 103 F F F
66302024-01-10T17:58:01.420Z 28 560-579 56 56 56 107 107 107 F F F
66312024-01-10T17:58:01.420Z 29 580-599 60 60 60 116 116 116 F F F
66322024-01-10T17:58:01.420ZMax gen: 61, Max flush: 120
66332024-01-10T17:58:01.420Z
66342024-01-10T17:58:01.420ZBegin loop 31
66352024-01-10T17:58:01.420ZDownstairs to restart: 2
66362024-01-10T17:58:01.420Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.780332076Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403}
66372024-01-10T17:58:01.421Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.780883594Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403}
66382024-01-10T17:58:01.421Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.780922742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403}
66392024-01-10T17:58:01.421Z{"msg":"Crucible 771f7956-66b9-47ae-927f-4d8e88a9e4bc has session id: dff822b1-bdc1-44b6-b508-45bb89d95501","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.780995456Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66402024-01-10T17:58:01.421Z{"msg":"Upstairs opts: Upstairs UUID: 771f7956-66b9-47ae-927f-4d8e88a9e4bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false"Crucible runtime is spawned
66412024-01-10T17:58:01.421Z,"v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.781153402Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66422024-01-10T17:58:01.421Z{"msg":"Crucible stats registered with UUID: 771f7956-66b9-47ae-927f-4d8e88a9e4bc","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.78121491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66432024-01-10T17:58:01.421Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.781257302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403}
66442024-01-10T17:58:01.423Z{"msg":"771f7956-66b9-47ae-927f-4d8e88a9e4bc active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.781427101Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66452024-01-10T17:58:01.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.781492373Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"0","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66462024-01-10T17:58:01.423Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.781635944Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"0","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66472024-01-10T17:58:01.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.781714531Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"1","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66482024-01-10T17:58:01.423Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.781811708Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"1","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66492024-01-10T17:58:01.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.781888177Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"2","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66502024-01-10T17:58:01.423Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.78196008Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"2","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66512024-01-10T17:58:01.424Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.782042985Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66522024-01-10T17:58:01.424Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.782134194Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66532024-01-10T17:58:01.424Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.782177336Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66542024-01-10T17:58:01.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.782217571Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66552024-01-10T17:58:01.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.782252928Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66562024-01-10T17:58:01.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.782276664Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66572024-01-10T17:58:01.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.783612802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"1","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66582024-01-10T17:58:01.424Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.78368695Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"1","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66592024-01-10T17:58:01.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.783736911Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"2","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66602024-01-10T17:58:01.424Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.783779287Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"2","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66612024-01-10T17:58:01.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.783834203Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"0","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66622024-01-10T17:58:01.424Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.784031156Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"0","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66632024-01-10T17:58:01.425Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.78473005Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"0","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66642024-01-10T17:58:01.428Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.784807004Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"1","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66652024-01-10T17:58:01.429Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.785116592Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"2","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66662024-01-10T17:58:01.429Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.785638671Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"1","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66672024-01-10T17:58:01.429Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.785698793Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"1","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66682024-01-10T17:58:01.429Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.785800834Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66692024-01-10T17:58:01.429Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.785823369Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"0","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66702024-01-10T17:58:01.429Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.785845871Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"0","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66712024-01-10T17:58:01.429Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.78586764Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66722024-01-10T17:58:01.429Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786008036Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"2","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66732024-01-10T17:58:01.429Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786066413Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"2","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66742024-01-10T17:58:01.430Z{"msg":"[0]R flush_numbers[0..12]: [91, 104, 112, 112, 109, 119, 117, 105, 120, 113, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786094098Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66752024-01-10T17:58:01.430Z{"msg":"[0]R generation[0..12]: [48, 55, 58, 58, 58, 60, 60, 56, 61, 59, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786135527Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66762024-01-10T17:58:01.430Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786178311Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66772024-01-10T17:58:01.431Z{"msg":"[1]R flush_numbers[0..12]: [91, 104, 112, 112, 109, 119, 117, 105, 120, 113, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786220663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66782024-01-10T17:58:01.431Z{"msg":"[1]R generation[0..12]: [48, 55, 58, 58, 58, 60, 60, 56, 61, 59, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.78626257Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66792024-01-10T17:58:01.431Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786305889Z","hostname":"Wait for a query_work_queue command to finish before sending IO
66802024-01-10T17:58:01.431Zip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66812024-01-10T17:58:01.431Z{"msg":"[2]R flush_numbers[0..12]: [91, 104, 112, 112, 109, 119, 117, 105, 120, 113, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786333241Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66822024-01-10T17:58:01.431Z{"msg":"[2]R generation[0..12]: [48, 55, 58, 58, 58, 60, 60, 56, 61, 59, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786373447Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66832024-01-10T17:58:01.431Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786399761Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66842024-01-10T17:58:01.432Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786440655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66852024-01-10T17:58:01.432Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786466052Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66862024-01-10T17:58:01.432Z{"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786506173Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66872024-01-10T17:58:01.432Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786549047Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"":"downstairs","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66882024-01-10T17:58:01.432Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786594049Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66892024-01-10T17:58:01.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786640751Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"0","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66902024-01-10T17:58:01.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786681791Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"1","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66912024-01-10T17:58:01.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786704852Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"client":"2","":"downstairs","session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66922024-01-10T17:58:01.432Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786744555Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66932024-01-10T17:58:01.432Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786784271Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66942024-01-10T17:58:01.436Z{"msg":"771f7956-66b9-47ae-927f-4d8e88a9e4bc is now active with session: dff822b1-bdc1-44b6-b508-45bb89d95501","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786833109Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66952024-01-10T17:58:01.436Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:57:56.786881358Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403,"session_id":"dff822b1-bdc1-44b6-b508-45bb89d95501"}
66962024-01-10T17:58:01.436Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
66972024-01-10T17:58:01.436Z,"time":"2024-01-10T17:57:56.786946598Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1403}
66982024-01-10T17:58:01.436ZLoading write count information from file "/tmp/test_repair_verify.data"
66992024-01-10T17:58:01.436ZRun Repair workload
67002024-01-10T17:58:01.437ZError: Difference in extent metadata found!
67012024-01-10T17:58:01.447Z01/30 Write block 243 len 12288 data: 6 9 8
67022024-01-10T17:58:01.447Z02/30 Read block 446 len 24576
67032024-01-10T17:58:01.447Z03/30 Write block 70 len 32768 data: 5 3 2 3 4 4 4 3
67042024-01-10T17:58:01.447Z04/30 Read block 478 len 40960
67052024-01-10T17:58:01.447Z05/30 Write block 62 len 24576 data: 3 3 6 7 8 5
67062024-01-10T17:58:01.447Z06/30 Write block 110 len 4096 data: 6
67072024-01-10T17:58:01.447Z07/30 Read block 209 len 20480
67082024-01-10T17:58:01.447Z08/30 Read block 195 len 8192
67092024-01-10T17:58:01.448Z09/30 Read block 205 len 24576
67102024-01-10T17:58:01.448Z10/30 Read block 364 len 32768
67112024-01-10T17:58:01.448Z11/30 Read block 521 len 36864
67122024-01-10T17:58:01.448Z12/30 Write block 17 len 20480 data: 7 8 7 5 6
67132024-01-10T17:58:01.449Z13/30 Write block 340 len 16384 data: 4 5 7 7
67142024-01-10T17:58:01.449Z14/30 Read block 576 len 24576
67152024-01-10T17:58:01.449Z15/30 Read block 255 len 8192
67162024-01-10T17:58:01.449Z16/30 Write block 304 len 4096 data: 6
67172024-01-10T17:58:01.449Z17/30 Read block 119 len 36864
67182024-01-10T17:58:01.449Z18/30 Read block 127 len 36864
67192024-01-10T17:58:01.449Z19/30 Read block 348 len 40960
67202024-01-10T17:58:01.450Z20/30 Read block 468 len 32768
67212024-01-10T17:58:01.450Z21/30 Write block 342 len 28672 data: 8 8 6 6 5 6 5
67222024-01-10T17:58:01.450Z22/30 Flush
67232024-01-10T17:58:01.450Z23/30 Flush
67242024-01-10T17:58:01.450Z24/30 Write block 205 len 12288 data: 4 3 3
67252024-01-10T17:58:01.450Z25/30 Write block 294 len 20480 data: 7 6 6 8 10
67262024-01-10T17:58:01.450Z26/30 Flush
67272024-01-10T17:58:01.450Z27/30 Write block 459 len 40960 data: 3 4 4 4 5 4 4 6 7 7
67282024-01-10T17:58:01.450Z28/30 Read block 159 len 8192
67292024-01-10T17:58:01.450Z29/30 Read block 402 len 16384
67302024-01-10T17:58:01.450Z30/30 Read block 188 len 8192
67312024-01-10T17:58:01.450Z----------------------------------------------------------------
67322024-01-10T17:58:01.450Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
67332024-01-10T17:58:01.450ZStates: Active Active Active
67342024-01-10T17:58:01.450ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67352024-01-10T17:58:01.451Z 1 Acked 1000 Write 3 Done Done Sent false
67362024-01-10T17:58:01.451Z 2 Acked 1001 Read 6 Done Done Sent false
67372024-01-10T17:58:01.451Z 3 Acked 1002 Write 8 Done Done Sent false
67382024-01-10T17:58:01.451Z 4 Acked 1003 Read 10 Done Done Sent false
67392024-01-10T17:58:01.451Z 5 Acked 1004 Write 6 Done Done Sent false
67402024-01-10T17:58:01.451Z 6 Acked 1005 Write 1 Done Done Sent false
67412024-01-10T17:58:01.451Z 7 Acked 1006 Read 5 Done Done Sent false
67422024-01-10T17:58:01.451Z 8 Acked 1007 Read 2 Done Done Sent false
67432024-01-10T17:58:01.451Z 9 Acked 1008 Read 6 Done Done Sent false
67442024-01-10T17:58:01.451Z 10 Acked 1009 Read 8 Done Done Sent false
67452024-01-10T17:58:01.451Z 11 Acked 1010 Read 9 Done Done Sent false
67462024-01-10T17:58:01.452Z 12 Acked 1011 Write 5 Done Done Sent false
67472024-01-10T17:58:01.452Z 13 Acked 1012 Write 4 Done Done Sent false
67482024-01-10T17:58:01.452Z 14 Acked 1013 Read 6 Done Done Sent false
67492024-01-10T17:58:01.452Z 15 Acked 1014 Read 2 Done Done Sent false
67502024-01-10T17:58:01.452Z 16 Acked 1015 Write 1 Done Done Sent false
67512024-01-10T17:58:01.452Z 17 Acked 1016 Read 9 Done Done Sent false
67522024-01-10T17:58:01.452Z 18 Acked 1017 Read 9 Done Done Sent false
67532024-01-10T17:58:01.452Z 19 Acked 1018 Read 10 Done Done Sent false
67542024-01-10T17:58:01.452Z 20 Acked 1019 Read 8 Done Done Sent false
67552024-01-10T17:58:01.452Z 21 Acked 1020 Write 7 Done Done Sent false
67562024-01-10T17:58:01.452Z 22 Acked 1021 Flush 0 Done Done Sent false
67572024-01-10T17:58:01.452Z 23 Acked 1022 Flush 0 Done Done Sent false
67582024-01-10T17:58:01.452Z 24 Acked 1023 Write 3 Done Done Sent false
67592024-01-10T17:58:01.452Z 25 Acked 1024 Write 5 Done Done Sent false
67602024-01-10T17:58:01.452Z 26 Acked 1025 Flush 0 Done Done Sent false
67612024-01-10T17:58:01.452Z 27 Acked 1026 Write 10 Done Done Sent false
67622024-01-10T17:58:01.453Z 28 Acked 1027 Read 2 Done Done Sent false
67632024-01-10T17:58:01.453Z 29 Acked 1028 Read 4 Done Done Sent false
67642024-01-10T17:58:01.453Z 30 Acked 1029 Read 2 Done Done Sent false
67652024-01-10T17:58:01.453Z STATES DS:0 DS:1 DS:2 TOTAL
67662024-01-10T17:58:01.453Z New 0 0 0 0
67672024-01-10T17:58:01.453Z Sent 0 0 30 30
67682024-01-10T17:58:01.453Z Done 30 30 0 60
67692024-01-10T17:58:01.453Z Skipped 0 0 0 0
67702024-01-10T17:58:01.453Z Error 0 0 0 0
67712024-01-10T17:58:01.453ZLast Flush: 1025 1025 0
67722024-01-10T17:58:01.453ZDownstairs last five completed:
67732024-01-10T17:58:01.453ZUpstairs last five completed: 30 29 28 27 26
67742024-01-10T17:58:01.453ZWrote out file "/tmp/test_repair_verify.data"
67752024-01-10T17:58:01.453Z
67762024-01-10T17:58:01.453ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
67772024-01-10T17:58:01.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
67782024-01-10T17:58:01.453Z 0 000-019 62 62 48 121 121 91 F F F <---
67792024-01-10T17:58:01.453Z 1 020-039 62 62 55 121 121 104 F F F <---
67802024-01-10T17:58:01.454Z 2 040-059 58 58 58 112 112 112 F F F
67812024-01-10T17:58:01.454Z 3 060-079 62 62 58 121 121 112 F F F <---
67822024-01-10T17:58:01.456Z 4 080-099 58 58 58 109 109 109 F F F
67832024-01-10T17:58:01.456Z 5 100-119 62 62 60 121 121 119 F F F <---
67842024-01-10T17:58:01.456Z 6 120-139 60 60 60 117 117 117 F F F
67852024-01-10T17:58:01.456Z 7 140-159 56 56 56 105 105 105 F F F
67862024-01-10T17:58:01.456Z 8 160-179 61 61 61 120 120 120 F F F
67872024-01-10T17:58:01.456Z 9 180-199 59 59 59 113 113 113 F F F
67882024-01-10T17:58:01.456Z 10 200-219 62 62 60 123 123 117 F F F <---
67892024-01-10T17:58:01.456Z 11 220-239 60 60 60 117 117 117 F F F
67902024-01-10T17:58:01.456Z 12 240-259 62 62 61 121 121 120 F F F <---
67912024-01-10T17:58:01.456Z 13 260-279 56 56 56 107 107 107 F F F
67922024-01-10T17:58:01.456Z 14 280-299 62 62 55 123 123 104 F F F <---
67932024-01-10T17:58:01.456Z 15 300-319 62 62 60 121 121 116 F F F <---
67942024-01-10T17:58:01.456Z 16 320-339 58 58 58 111 111 111 F F F
67952024-01-10T17:58:01.456Z 17 340-359 62 62 53 121 121 101 F F F <---
67962024-01-10T17:58:01.456Z 18 360-379 58 58 58 111 111 111 F F F
67972024-01-10T17:58:01.456Z 19 380-399 58 58 58 112 112 112 F F F
67982024-01-10T17:58:01.456Z 20 400-419 60 60 60 114 114 114 F F F
67992024-01-10T17:58:01.456Z 21 420-439 59 59 59 113 113 113 F F F
68002024-01-10T17:58:01.456Z 22 440-459 60 60 60 114 114 114 T T F <---
68012024-01-10T17:58:01.476Z 23 460-479 60 60 60 119 119 119 T T F <---
68022024-01-10T17:58:01.476Z 24 480-499 58 58 58 109 109 109 F F F
68032024-01-10T17:58:01.476Z 25 500-519 60 60 60 117 117 117 F F F
68042024-01-10T17:58:01.476Z 26 520-539 60 60 60 116 116 116 F F F
68052024-01-10T17:58:01.476Z 27 540-559 54 54 54 103 103 103 F F F
68062024-01-10T17:58:01.478Z 28 560-579 56 56 56 107 107 107 F F F
68072024-01-10T17:58:01.478Z 29 580-599 60 60 60 116 116 116 F F F
68082024-01-10T17:58:01.478ZMax gen: 62, Max flush: 123
68092024-01-10T17:58:01.478ZA Difference in extent metadata is expected here
68102024-01-10T17:58:01.478ZOn loop 31
68112024-01-10T17:58:01.478Z
68122024-01-10T17:58:01.478ZVerifying data now
68132024-01-10T17:58:02.001ZNo change after verify
68142024-01-10T17:58:02.001ZLoop: 31 Downstairs dump after verify (and repair):
68152024-01-10T17:58:02.022ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68162024-01-10T17:58:02.022Z 0 000-019 62 62 62 121 121 121 F F F
68172024-01-10T17:58:02.022Z 1 020-039 62 62 62 121 121 121 F F F
68182024-01-10T17:58:02.022Z 2 040-059 58 58 58 112 112 112 F F F
68192024-01-10T17:58:02.022Z 3 060-079 62 62 62 121 121 121 F F F
68202024-01-10T17:58:02.022Z 4 080-099 58 58 58 109 109 109 F F F
68212024-01-10T17:58:02.022Z 5 100-119 62 62 62 121 121 121 F F F
68222024-01-10T17:58:02.030Z 6 120-139 60 60 60 117 117 117 F F F
68232024-01-10T17:58:02.030Z 7 140-159 56 56 56 105 105 105 F F F
68242024-01-10T17:58:02.030Z 8 160-179 61 61 61 120 120 120 F F F
68252024-01-10T17:58:02.030Z 9 180-199 59 59 59 113 113 113 F F F
68262024-01-10T17:58:02.030Z 10 200-219 62 62 62 123 123 123 F F F
68272024-01-10T17:58:02.030Z 11 220-239 60 60 60 117 117 117 F F F
68282024-01-10T17:58:02.030Z 12 240-259 62 62 62 121 121 121 F F F
68292024-01-10T17:58:02.030Z 13 260-279 56 56 56 107 107 107 F F F
68302024-01-10T17:58:02.030Z 14 280-299 62 62 62 123 123 123 F F F
68312024-01-10T17:58:02.032Z 15 300-319 62 62 62 121 121 121 F F F
68322024-01-10T17:58:02.032Z 16 320-339 58 58 58 111 111 111 F F F
68332024-01-10T17:58:02.032Z 17 340-359 62 62 62 121 121 121 F F F
68342024-01-10T17:58:02.032Z 18 360-379 58 58 58 111 111 111 F F F
68352024-01-10T17:58:02.032Z 19 380-399 58 58 58 112 112 112 F F F
68362024-01-10T17:58:02.032Z 20 400-419 60 60 60 114 114 114 F F F
68372024-01-10T17:58:02.032Z 21 420-439 59 59 59 113 113 113 F F F
68382024-01-10T17:58:02.032Z 22 440-459 63 63 63 124 124 124 F F F
68392024-01-10T17:58:02.032Z 23 460-479 63 63 63 124 124 124 F F F
68402024-01-10T17:58:02.032Z 24 480-499 58 58 58 109 109 109 F F F
68412024-01-10T17:58:02.032Z 25 500-519 60 60 60 117 117 117 F F F
68422024-01-10T17:58:02.032Z 26 520-539 60 60 60 116 116 116 F F F
68432024-01-10T17:58:02.032Z 27 540-559 54 54 54 103 103 103 F F F
68442024-01-10T17:58:02.032Z 28 560-579 56 56 56 107 107 107 F F F
68452024-01-10T17:58:02.032Z 29 580-599 60 60 60 116 116 116 F F F
68462024-01-10T17:58:02.032ZMax gen: 63, Max flush: 124
68472024-01-10T17:58:02.032Z
68482024-01-10T17:58:02.033ZBegin loop 32
68492024-01-10T17:58:02.033ZDownstairs to restart: 1
68502024-01-10T17:58:02.073Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.057426255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411}
68512024-01-10T17:58:02.089Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.057967048Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411}
68522024-01-10T17:58:02.089Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.05803359Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411}
68532024-01-10T17:58:02.089Z{Crucible runtime is spawned
68542024-01-10T17:58:02.089Z"msg":"Crucible 55595731-ba83-4571-9e46-cd2784c7da2a has session id: 8c26a06e-2e65-42f8-9f19-d05f8a1e2a87","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058095857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68552024-01-10T17:58:02.089Z{"msg":"Upstairs opts: Upstairs UUID: 55595731-ba83-4571-9e46-cd2784c7da2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.05815192Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68562024-01-10T17:58:02.089Z{"msg":"Crucible stats registered with UUID: 55595731-ba83-4571-9e46-cd2784c7da2a","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058226021Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68572024-01-10T17:58:02.089Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.05826855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411}
68582024-01-10T17:58:02.090Z{"msg":"55595731-ba83-4571-9e46-cd2784c7da2a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058383235Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68592024-01-10T17:58:02.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058434948Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"0","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68602024-01-10T17:58:02.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058461916Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"0","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68612024-01-10T17:58:02.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058547079Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"1","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68622024-01-10T17:58:02.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058591566Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"1","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68632024-01-10T17:58:02.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058632331Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"2","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68642024-01-10T17:58:02.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058673093Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"2","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68652024-01-10T17:58:02.090Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058710675Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68662024-01-10T17:58:02.090Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058749167Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68672024-01-10T17:58:02.090Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058823401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68682024-01-10T17:58:02.090Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.058977397Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68692024-01-10T17:58:02.090Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.059082609Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68702024-01-10T17:58:02.090Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.059144364Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68712024-01-10T17:58:02.090Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.060208425Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"2","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68722024-01-10T17:58:02.091Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.060273939Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"2","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68732024-01-10T17:58:02.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.060316184Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"0","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68742024-01-10T17:58:02.092Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.060357478Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"0","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68752024-01-10T17:58:02.092Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.061327137Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"2","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68762024-01-10T17:58:02.092Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.061453062Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"0","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68772024-01-10T17:58:02.092Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.061603476Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"1","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68782024-01-10T17:58:02.092Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.061673798Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"1","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68792024-01-10T17:58:02.092Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.062454745Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"2","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68802024-01-10T17:58:02.092Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.062494528Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"2","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68812024-01-10T17:58:02.092Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.062517717Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68822024-01-10T17:58:02.093Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.062579895Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"0","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68832024-01-10T17:58:02.093Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.06262319Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"0","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68842024-01-10T17:58:02.093Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.062692077Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68852024-01-10T17:58:02.093Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.06273466Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"1","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68862024-01-10T17:58:02.093Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063427433Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"1","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68872024-01-10T17:58:02.093Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.06348602Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"1","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68882024-01-10T17:58:02.093Z{"msg":"[0]R flush_numbers[0..12]: [121, 121, 112, 121, 109, 121, 117, 105, 120, 113, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063512551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68892024-01-10T17:58:02.093Z{"msg":"[0]R generation[0..12]: [62, 62, 58, 62, 58, 62, 60, 56, 61, 59, 62, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.06353631Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68902024-01-10T17:58:02.093Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063578774Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68912024-01-10T17:58:02.093Z{"msg":"[1]R flush_numbers[0..12]: [121, 121, 112, 121, 109, 121, 117, 105, 120, 113, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.06360382Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68922024-01-10T17:58:02.093Z{"msg":"[1]R generation[0..12]: [62, 62, 58, 62, 58, 62, 60, 56, 61, 59, 62, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063652682Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68932024-01-10T17:58:02.093Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063677867Z"Wait for a query_work_queue command to finish before sending IO
68942024-01-10T17:58:02.093Z,"hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68952024-01-10T17:58:02.093Z{"msg":"[2]R flush_numbers[0..12]: [121, 121, 112, 121, 109, 121, 117, 105, 120, 113, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063706069Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68962024-01-10T17:58:02.093Z{"msg":"[2]R generation[0..12]: [62, 62, 58, 62, 58, 62, 60, 56, 61, 59, 62, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.06373024Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68972024-01-10T17:58:02.093Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063753684Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68982024-01-10T17:58:02.094Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063793557Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
68992024-01-10T17:58:02.094Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063817019Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69002024-01-10T17:58:02.094Z{"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063837994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69012024-01-10T17:58:02.094Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.06385918Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"":"downstairs","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69022024-01-10T17:58:02.094Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063880528Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69032024-01-10T17:58:02.094Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063917867Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"0","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69042024-01-10T17:58:02.094Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063940942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"1","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69052024-01-10T17:58:02.094Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.063963766Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"client":"2","":"downstairs","session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69062024-01-10T17:58:02.095Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.064011208Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69072024-01-10T17:58:02.095Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.06403264Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69082024-01-10T17:58:02.095Z{"msg":"55595731-ba83-4571-9e46-cd2784c7da2a is now active with session: 8c26a06e-2e65-42f8-9f19-d05f8a1e2a87","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.064054032Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69092024-01-10T17:58:02.095Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.064074643Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411,"session_id":"8c26a06e-2e65-42f8-9f19-d05f8a1e2a87"}
69102024-01-10T17:58:02.095Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:02.064095129Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1411}
69112024-01-10T17:58:02.095ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69122024-01-10T17:58:02.095ZLoading write count information from file "/tmp/test_repair_verify.data"
69132024-01-10T17:58:02.095ZRun Repair workload
69142024-01-10T17:58:02.096Z01/30 Write block 555 len 32768 data: 6 4 5 4 3 4 6 7
69152024-01-10T17:58:02.096Z02/30 Read block 81 len 16384
69162024-01-10T17:58:02.096Z03/30 Write block 165 len 20480 data: 11 11 10 8 7
69172024-01-10T17:58:02.097Z04/30 Flush
69182024-01-10T17:58:02.097Z05/30 Write block 315 len 36864 data: 5 6 4 4 5 5 5 5 5
69192024-01-10T17:58:02.097Z06/30 Read block 21 len 24576
69202024-01-10T17:58:02.097Z07/30 Write block 410 len 16384 data: 8 8 7 7
69212024-01-10T17:58:02.097Z08/30 Write block 547 len 32768 data: 9 9 9 8 7 10 9 8
69222024-01-10T17:58:02.097Z09/30 Write block 575 len 28672 data: 7 8 7 6 6 6 6
69232024-01-10T17:58:02.097Z10/30 Write block 104 len 40960 data: 3 4 4 3 4 5 7 7 8 8
69242024-01-10T17:58:02.097Z11/30 Flush
69252024-01-10T17:58:02.106Z12/30 Write block 274 len 40960 data: 8 9 8 7 6 7 6 6 6 7
69262024-01-10T17:58:02.106Z13/30 Read block 475 len 16384
69272024-01-10T17:58:02.106Z14/30 Read block 433 len 40960
69282024-01-10T17:58:02.106Z15/30 Read block 541 len 4096
69292024-01-10T17:58:02.107Z16/30 Read block 165 len 4096
69302024-01-10T17:58:02.107Z17/30 Read block 350 len 24576
69312024-01-10T17:58:02.107Z18/30 Read block 507 len 24576
69322024-01-10T17:58:02.107Z19/30 Read block 183 len 4096
69332024-01-10T17:58:02.107Z20/30 Read block 312 len 24576
69342024-01-10T17:58:02.130Z21/30 Read block 486 len 32768
69352024-01-10T17:58:02.130Z22/30 Write block 193 len 8192 data: 8 9
69362024-01-10T17:58:02.130Z23/30 Read block 420 len 4096
69372024-01-10T17:58:02.130Z24/30 Read block 272 len 8192
69382024-01-10T17:58:02.130Z25/30 Write block 452 len 12288 data: 5 6 5
69392024-01-10T17:58:02.130Z26/30 Write block 403 len 32768 data: 6 6 8 9 9 7 6 9
69402024-01-10T17:58:02.132Z27/30 Read block 204 len 20480
69412024-01-10T17:58:02.132Z28/30 Read block 333 len 20480
69422024-01-10T17:58:02.132Z29/30 Write block 146 len 8192 data: 5 5
69432024-01-10T17:58:02.132Z30/30 Read block 367 len 40960
69442024-01-10T17:58:02.132Z----------------------------------------------------------------
69452024-01-10T17:58:02.132Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:26
69462024-01-10T17:58:02.132ZStates: Active Active Active
69472024-01-10T17:58:02.132ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69482024-01-10T17:58:02.132Z 5 Acked 1004 Write 9 Done Done Done false
69492024-01-10T17:58:02.132Z 6 Acked 1005 Read 6 Done Sent Done false
69502024-01-10T17:58:02.132Z 7 Acked 1006 Write 4 Done Sent Done false
69512024-01-10T17:58:02.132Z 8 Acked 1007 Write 8 Done Sent Done false
69522024-01-10T17:58:02.133Z 9 Acked 1008 Write 7 Done Sent Done false
69532024-01-10T17:58:02.133Z 10 Acked 1009 Write 10 Done Sent Done false
69542024-01-10T17:58:02.133Z 11 Acked 1010 Flush 0 Done Sent Done false
69552024-01-10T17:58:02.133Z 12 Acked 1011 Write 10 Done Sent Done false
69562024-01-10T17:58:02.133Z 13 Acked 1012 Read 4 Done Sent Done false
69572024-01-10T17:58:02.133Z 14 Acked 1013 Read 10 Done Sent Done false
69582024-01-10T17:58:02.133Z 15 Acked 1014 Read 1 Done Sent Done false
69592024-01-10T17:58:02.133Z 16 Acked 1015 Read 1 Done Sent Done false
69602024-01-10T17:58:02.133Z 17 Acked 1016 Read 6 Done Sent Done false
69612024-01-10T17:58:02.133Z 18 Acked 1017 Read 6 Done Sent Done false
69622024-01-10T17:58:02.133Z 19 Acked 1018 Read 1 Done Sent Done false
69632024-01-10T17:58:02.133Z 20 Acked 1019 Read 6 Done Sent Done false
69642024-01-10T17:58:02.134Z 21 Acked 1020 Read 8 Done Sent Done false
69652024-01-10T17:58:02.134Z 22 Acked 1021 Write 2 Done Sent Done false
69662024-01-10T17:58:02.135Z 23 Acked 1022 Read 1 Done Sent Done false
69672024-01-10T17:58:02.135Z 24 Acked 1023 Read 2 Done Sent Done false
69682024-01-10T17:58:02.135Z 25 Acked 1024 Write 3 Done Sent Done false
69692024-01-10T17:58:02.135Z 26 Acked 1025 Write 8 Done Sent Done false
69702024-01-10T17:58:02.135Z 27 Acked 1026 Read 5 Done Sent Done false
69712024-01-10T17:58:02.135Z 28 Acked 1027 Read 5 Done Sent Done false
69722024-01-10T17:58:02.135Z 29 Acked 1028 Write 2 Done Sent Done false
69732024-01-10T17:58:02.135Z 30 Acked 1029 Read 10 Done Sent Done false
69742024-01-10T17:58:02.135Z STATES DS:0 DS:1 DS:2 TOTAL
69752024-01-10T17:58:02.135Z New 0 0 0 0
69762024-01-10T17:58:02.135Z Sent 0 25 0 25
69772024-01-10T17:58:02.135Z Done 26 1 26 53
69782024-01-10T17:58:02.135Z Skipped 0 0 0 0
69792024-01-10T17:58:02.135Z Error 0 0 0 0
69802024-01-10T17:58:02.135ZLast Flush: 1010 1003 1010
69812024-01-10T17:58:02.136ZDownstairs last five completed: 1003 1002 1001 1000
69822024-01-10T17:58:02.136ZUpstairs last five completed: 30 29 28 27 26
69832024-01-10T17:58:02.136ZWrote out file "/tmp/test_repair_verify.data"
69842024-01-10T17:58:02.142Z
69852024-01-10T17:58:02.142ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
69862024-01-10T17:58:02.167ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
69872024-01-10T17:58:02.167Z 0 000-019 62 62 62 121 121 121 F F F
69882024-01-10T17:58:02.187Z 1 020-039 62 62 62 121 121 121 F F F
69892024-01-10T17:58:02.187Z 2 040-059 58 58 58 112 112 112 F F F
69902024-01-10T17:58:02.187Z 3 060-079 62 62 62 121 121 121 F F F
69912024-01-10T17:58:02.187Z 4 080-099 58 58 58 109 109 109 F F F
69922024-01-10T17:58:02.188Z 5 100-119 64 62 64 126 121 126 F F F <---
69932024-01-10T17:58:02.188Z 6 120-139 60 60 60 117 117 117 F F F
69942024-01-10T17:58:02.188Z 7 140-159 56 56 56 105 105 105 T F T <---
69952024-01-10T17:58:02.188Z 8 160-179 64 64 64 125 125 125 F F F
69962024-01-10T17:58:02.189Z 9 180-199 59 59 59 113 113 113 T F T <---
69972024-01-10T17:58:02.189Z 10 200-219 62 62 62 123 123 123 F F F
69982024-01-10T17:58:02.189Z 11 220-239 60 60 60 117 117 117 F F F
69992024-01-10T17:58:02.191Z 12 240-259 62 62 62 121 121 121 F F F
70002024-01-10T17:58:02.191Z 13 260-279 56 56 56 107 107 107 T F T <---
70012024-01-10T17:58:02.191Z 14 280-299 62 62 62 123 123 123 T F T <---
70022024-01-10T17:58:02.191Z 15 300-319 64 62 64 126 121 126 F T F <---
70032024-01-10T17:58:02.191Z 16 320-339 64 58 64 126 111 126 F T F <---
70042024-01-10T17:58:02.191Z 17 340-359 62 62 62 121 121 121 F F F
70052024-01-10T17:58:02.191Z 18 360-379 58 58 58 111 111 111 F F F
70062024-01-10T17:58:02.191Z 19 380-399 58 58 58 112 112 112 F F F
70072024-01-10T17:58:02.192Z 20 400-419 64 60 64 126 114 126 T F T <---
70082024-01-10T17:58:02.192Z 21 420-439 59 59 59 113 113 113 F F F
70092024-01-10T17:58:02.192Z 22 440-459 63 63 63 124 124 124 T F T <---
70102024-01-10T17:58:02.192Z 23 460-479 63 63 63 124 124 124 F F F
70112024-01-10T17:58:02.192Z 24 480-499 58 58 58 109 109 109 F F F
70122024-01-10T17:58:02.193Z 25 500-519 60 60 60 117 117 117 F F F
70132024-01-10T17:58:02.194Z 26 520-539 60 60 60 116 116 116 F F F
70142024-01-10T17:58:02.194Z 27 540-559 64 64 64 126 125 126 F F F <---
70152024-01-10T17:58:02.194Z 28 560-579 64 64 64 126 125 126 F F F <---
70162024-01-10T17:58:02.194Z 29 580-599 64 60 64 126 116 126 F F F <---
70172024-01-10T17:58:02.194ZMax gen: 64, Max flush: 126
70182024-01-10T17:58:02.194ZError: Difference in extent metadata found!
70192024-01-10T17:58:02.194ZA Difference in extent metadata is expected here
70202024-01-10T17:58:02.194ZOn loop 32
70212024-01-10T17:58:02.194Z
70222024-01-10T17:58:02.194ZVerifying data now
70232024-01-10T17:58:07.375ZNo change after verify
70242024-01-10T17:58:07.375ZLoop: 32 Downstairs dump after verify (and repair):
70252024-01-10T17:58:07.397ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70262024-01-10T17:58:07.398Z 0 000-019 62 62 62 121 121 121 F F F
70272024-01-10T17:58:07.398Z 1 020-039 62 62 62 121 121 121 F F F
70282024-01-10T17:58:07.398Z 2 040-059 58 58 58 112 112 112 F F F
70292024-01-10T17:58:07.398Z 3 060-079 62 62 62 121 121 121 F F F
70302024-01-10T17:58:07.398Z 4 080-099 58 58 58 109 109 109 F F F
70312024-01-10T17:58:07.401Z 5 100-119 64 64 64 126 126 126 F F F
70322024-01-10T17:58:07.401Z 6 120-139 60 60 60 117 117 117 F F F
70332024-01-10T17:58:07.401Z 7 140-159 65 65 65 127 127 127 F F F
70342024-01-10T17:58:07.401Z 8 160-179 64 64 64 125 125 125 F F F
70352024-01-10T17:58:07.402Z 9 180-199 65 65 65 127 127 127 F F F
70362024-01-10T17:58:07.402Z 10 200-219 62 62 62 123 123 123 F F F
70372024-01-10T17:58:07.402Z 11 220-239 60 60 60 117 117 117 F F F
70382024-01-10T17:58:07.402Z 12 240-259 62 62 62 121 121 121 F F F
70392024-01-10T17:58:07.402Z 13 260-279 65 65 65 127 127 127 F F F
70402024-01-10T17:58:07.402Z 14 280-299 65 65 65 127 127 127 F F F
70412024-01-10T17:58:07.402Z 15 300-319 64 64 64 126 126 126 F F F
70422024-01-10T17:58:07.402Z 16 320-339 64 64 64 126 126 126 F F F
70432024-01-10T17:58:07.402Z 17 340-359 62 62 62 121 121 121 F F F
70442024-01-10T17:58:07.402Z 18 360-379 58 58 58 111 111 111 F F F
70452024-01-10T17:58:07.402Z 19 380-399 58 58 58 112 112 112 F F F
70462024-01-10T17:58:07.402Z 20 400-419 65 65 65 127 127 127 F F F
70472024-01-10T17:58:07.402Z 21 420-439 59 59 59 113 113 113 F F F
70482024-01-10T17:58:07.402Z 22 440-459 65 65 65 127 127 127 F F F
70492024-01-10T17:58:07.402Z 23 460-479 63 63 63 124 124 124 F F F
70502024-01-10T17:58:07.402Z 24 480-499 58 58 58 109 109 109 F F F
70512024-01-10T17:58:07.402Z 25 500-519 60 60 60 117 117 117 F F F
70522024-01-10T17:58:07.402Z 26 520-539 60 60 60 116 116 116 F F F
70532024-01-10T17:58:07.402Z 27 540-559 64 64 64 126 126 126 F F F
70542024-01-10T17:58:07.402Z 28 560-579 64 64 64 126 126 126 F F F
70552024-01-10T17:58:07.402Z 29 580-599 64 64 64 126 126 126 F F F
70562024-01-10T17:58:07.402ZMax gen: 65, Max flush: 127
70572024-01-10T17:58:07.406Z
70582024-01-10T17:58:07.406ZBegin loop 33
70592024-01-10T17:58:07.406ZDownstairs to restart: 2
70602024-01-10T17:58:07.432Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.433740678Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419}
70612024-01-10T17:58:07.433Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.434322778Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419}
70622024-01-10T17:58:07.440Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.434393177Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419}
70632024-01-10T17:58:07.440Z{"msg":"Crucible cdd159ad-7be6-458f-a71c-5ef521e2dd51 has session id: e0e552e6-9eb2-4e12-b9f3-027b7b1d8646","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.434436926Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70642024-01-10T17:58:07.440Z{"msg":"Upstairs opts: Upstairs UUID: cdd159ad-7be6-458f-a71c-5ef521e2dd51, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.434521804Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70652024-01-10T17:58:07.442Z{"msg":"Crucible runtime is spawned
70662024-01-10T17:58:07.442ZCrucible stats registered with UUID: cdd159ad-7be6-458f-a71c-5ef521e2dd51","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.434699338Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70672024-01-10T17:58:07.442Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.434820577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419}
70682024-01-10T17:58:07.442Z{"msg":"cdd159ad-7be6-458f-a71c-5ef521e2dd51 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435055457Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70692024-01-10T17:58:07.443Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.43511651Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"0","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70702024-01-10T17:58:07.443Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435143019Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"0","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70712024-01-10T17:58:07.443Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435170138Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"1","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70722024-01-10T17:58:07.443Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435259349Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"1","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70732024-01-10T17:58:07.443Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435342673Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"2","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70742024-01-10T17:58:07.443Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435458825Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"2","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70752024-01-10T17:58:07.443Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435561786Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70762024-01-10T17:58:07.443Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435607915Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70772024-01-10T17:58:07.443Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435652886Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70782024-01-10T17:58:07.443Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435697208Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70792024-01-10T17:58:07.443Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435734342Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70802024-01-10T17:58:07.444Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.435763182Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70812024-01-10T17:58:07.444Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.436665841Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"1","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70822024-01-10T17:58:07.444Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.43675326Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"1","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70832024-01-10T17:58:07.444Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.436779777Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"0","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70842024-01-10T17:58:07.444Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.436802263Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"0","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70852024-01-10T17:58:07.444Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.437262439Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"2","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70862024-01-10T17:58:07.444Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.437340547Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"2","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70872024-01-10T17:58:07.444Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.437893861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"1","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70882024-01-10T17:58:07.462Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.438045706Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"0","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70892024-01-10T17:58:07.462Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.438418973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"2","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70902024-01-10T17:58:07.462Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439094271Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"1","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70912024-01-10T17:58:07.462Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439184087Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"1","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70922024-01-10T17:58:07.463Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439324399Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70932024-01-10T17:58:07.463Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439355775Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"0","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70942024-01-10T17:58:07.463Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439423975Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"0","":"downstairs","Wait for a query_work_queue command to finish before sending IO
70952024-01-10T17:58:07.463Zsession_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70962024-01-10T17:58:07.463Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439458039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70972024-01-10T17:58:07.464Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439478401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"2","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70982024-01-10T17:58:07.464Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439527805Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"2","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
70992024-01-10T17:58:07.464Z{"msg":"[0]R flush_numbers[0..12]: [121, 121, 112, 121, 109, 126, 117, 127, 125, 127, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439552666Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71002024-01-10T17:58:07.464Z{"msg":"[0]R generation[0..12]: [62, 62, 58, 62, 58, 64, 60, 65, 64, 65, 62, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439575606Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71012024-01-10T17:58:07.464Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.43960148Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71022024-01-10T17:58:07.464Z{"msg":"[1]R flush_numbers[0..12]: [121, 121, 112, 121, 109, 126, 117, 127, 125, 127, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439643068Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71032024-01-10T17:58:07.464Z{"msg":"[1]R generation[0..12]: [62, 62, 58, 62, 58, 64, 60, 65, 64, 65, 62, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439666756Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71042024-01-10T17:58:07.464Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.43969032Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71052024-01-10T17:58:07.464Z{"msg":"[2]R flush_numbers[0..12]: [121, 121, 112, 121, 109, 126, 117, 127, 125, 127, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439731353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71062024-01-10T17:58:07.464Z{"msg":"[2]R generation[0..12]: [62, 62, 58, 62, 58, 64, 60, 65, 64, 65, 62, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439755258Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71072024-01-10T17:58:07.464Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439796256Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71082024-01-10T17:58:07.464Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439819041Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71092024-01-10T17:58:07.464Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439858579Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71102024-01-10T17:58:07.464Z{"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439880894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71112024-01-10T17:58:07.464Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439920057Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"":"downstairs","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71122024-01-10T17:58:07.464Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439951565Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71132024-01-10T17:58:07.464Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.439998588Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"0","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71142024-01-10T17:58:07.464Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.440021987Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"1","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71152024-01-10T17:58:07.465Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.440061294Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"client":"2","":"downstairs","session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71162024-01-10T17:58:07.465Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.440083997Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71172024-01-10T17:58:07.465Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.440103971Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71182024-01-10T17:58:07.465Zip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71192024-01-10T17:58:07.465Z{"msg":"cdd159ad-7be6-458f-a71c-5ef521e2dd51 is now active with session: e0e552e6-9eb2-4e12-b9f3-027b7b1d8646","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.440207685Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71202024-01-10T17:58:07.465Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.440233458Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419,"session_id":"e0e552e6-9eb2-4e12-b9f3-027b7b1d8646"}
71212024-01-10T17:58:07.465Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:07.440272362Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1419}
71222024-01-10T17:58:07.465ZLoading write count information from file "/tmp/test_repair_verify.data"
71232024-01-10T17:58:07.465ZRun Repair workload
71242024-01-10T17:58:07.466Z01/30 Write block 75 len 16384 data: 5 5 4 3
71252024-01-10T17:58:07.466Z02/30 Write block 528 len 40960 data: 5 7 8 10 8 8 7 6 7 7
71262024-01-10T17:58:07.466Z03/30 Write block 58 len 12288 data: 5 5 5
71272024-01-10T17:58:07.466Z04/30 Read block 489 len 4096
71282024-01-10T17:58:07.466Z05/30 Write block 34 len 4096 data: 5
71292024-01-10T17:58:07.466Z06/30 Flush
71302024-01-10T17:58:07.466Z07/30 Write block 289 len 40960 data: 6 9 11 10 8 8 7 7 9 11
71312024-01-10T17:58:07.467Z08/30 Read block 519 len 12288
71322024-01-10T17:58:07.472Z09/30 Write block 447 len 8192 data: 9 7
71332024-01-10T17:58:07.473Z10/30 Write block 4 len 36864 data: 6 4 4 3 3 3 4 4 6
71342024-01-10T17:58:07.473Z11/30 Read block 43 len 32768
71352024-01-10T17:58:07.473Z12/30 Write block 247 len 8192 data: 5 5
71362024-01-10T17:58:07.473Z13/30 Read block 203 len 36864
71372024-01-10T17:58:07.473Z14/30 Read block 367 len 28672
71382024-01-10T17:58:07.473Z15/30 Read block 293 len 36864
71392024-01-10T17:58:07.473Z16/30 Write block 193 len 32768 data: 9 10 9 6 6 7 7 7
71402024-01-10T17:58:07.473Z17/30 Write block 281 len 4096 data: 7
71412024-01-10T17:58:07.473Z18/30 Read block 142 len 16384
71422024-01-10T17:58:07.477Z19/30 Read block 184 len 8192
71432024-01-10T17:58:07.477Z20/30 Read block 281 len 12288
71442024-01-10T17:58:07.477Z21/30 Read block 391 len 4096
71452024-01-10T17:58:07.477Z22/30 Write block 187 len 32768 data: 8 9 9 9 9 9 10 11
71462024-01-10T17:58:07.477Z23/30 Read block 412 len 12288
71472024-01-10T17:58:07.481Z24/30 Write block 1 len 28672 data: 4 5 6 7 5 5 4
71482024-01-10T17:58:07.481Z25/30 Read block 47 len 8192
71492024-01-10T17:58:07.485Z26/30 Read block 219 len 28672
71502024-01-10T17:58:07.488Z27/30 Read block 357 len 40960
71512024-01-10T17:58:07.488Z28/30 Write block 219 len 12288 data: 6 6 6
71522024-01-10T17:58:07.492Z29/30 Read block 431 len 4096
71532024-01-10T17:58:07.492Z30/30 Read block 489 len 36864
71542024-01-10T17:58:07.496Z----------------------------------------------------------------
71552024-01-10T17:58:07.496Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
71562024-01-10T17:58:07.496ZStates: Active Active Active
71572024-01-10T17:58:07.496ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
71582024-01-10T17:58:07.496Z 1 Acked 1000 Write 4 Done Done Sent false
71592024-01-10T17:58:07.496Z 2 Acked 1001 Write 10 Done Done Sent false
71602024-01-10T17:58:07.497Z 3 Acked 1002 Write 3 Done Done Sent false
71612024-01-10T17:58:07.497Z 4 Acked 1003 Read 1 Done Done Sent false
71622024-01-10T17:58:07.497Z 5 Acked 1004 Write 1 Done Done Sent false
71632024-01-10T17:58:07.497Z 6 Acked 1005 Flush 0 Done Done Sent false
71642024-01-10T17:58:07.497Z 7 Acked 1006 Write 10 Done Done Sent false
71652024-01-10T17:58:07.497Z 8 Acked 1007 Read 3 Done Done Sent false
71662024-01-10T17:58:07.497Z 9 Acked 1008 Write 2 Done Done Sent false
71672024-01-10T17:58:07.497Z 10 Acked 1009 Write 9 Done Done Sent false
71682024-01-10T17:58:07.497Z 11 Acked 1010 Read 8 Done Done Sent false
71692024-01-10T17:58:07.497Z 12 Acked 1011 Write 2 Done Done Sent false
71702024-01-10T17:58:07.497Z 13 Acked 1012 Read 9 Done Done Sent false
71712024-01-10T17:58:07.497Z 14 Acked 1013 Read 7 Done Done Sent false
71722024-01-10T17:58:07.497Z 15 Acked 1014 Read 9 Done Done Sent false
71732024-01-10T17:58:07.498Z 16 Acked 1015 Write 8 Done Done Sent false
71742024-01-10T17:58:07.498Z 17 Acked 1016 Write 1 Done Done Sent false
71752024-01-10T17:58:07.498Z 18 Acked 1017 Read 4 Done Done Sent false
71762024-01-10T17:58:07.498Z 19 Acked 1018 Read 2 Done Done Sent false
71772024-01-10T17:58:07.498Z 20 Acked 1019 Read 3 Done Done Sent false
71782024-01-10T17:58:07.498Z 21 Acked 1020 Read 1 Done Done Sent false
71792024-01-10T17:58:07.498Z 22 Acked 1021 Write 8 Done Done Sent false
71802024-01-10T17:58:07.498Z 23 Acked 1022 Read 3 Done Done Sent false
71812024-01-10T17:58:07.498Z 24 Acked 1023 Write 7 Done Done Sent false
71822024-01-10T17:58:07.498Z 25 Acked 1024 Read 2 Done Done Sent false
71832024-01-10T17:58:07.498Z 26 Acked 1025 Read 7 Done Done Sent false
71842024-01-10T17:58:07.498Z 27 Acked 1026 Read 10 Done Done Sent false
71852024-01-10T17:58:07.499Z 28 Acked 1027 Write 3 Done Done Sent false
71862024-01-10T17:58:07.499Z 29 Acked 1028 Read 1 Done Done Sent false
71872024-01-10T17:58:07.499Z 30 Acked 1029 Read 9 Done Sent Sent false
71882024-01-10T17:58:07.500Z STATES DS:0 DS:1 DS:2 TOTAL
71892024-01-10T17:58:07.500Z New 0 0 0 0
71902024-01-10T17:58:07.500Z Sent 0 1 30 31
71912024-01-10T17:58:07.500Z Done 30 29 0 59
71922024-01-10T17:58:07.500Z Skipped 0 0 0 0
71932024-01-10T17:58:07.500Z Error 0 0 0 0
71942024-01-10T17:58:07.500ZLast Flush: 1005 1005 0
71952024-01-10T17:58:07.500ZDownstairs last five completed:
71962024-01-10T17:58:07.500ZUpstairs last five completed: 30 29 28 27 26
71972024-01-10T17:58:07.500ZWrote out file "/tmp/test_repair_verify.data"
71982024-01-10T17:58:07.505Z
71992024-01-10T17:58:07.508ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72002024-01-10T17:58:07.531ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72012024-01-10T17:58:07.531Z 0 000-019 62 62 62 121 121 121 T T F <---
72022024-01-10T17:58:07.531Z 1 020-039 66 66 62 128 128 121 F F F <---
72032024-01-10T17:58:07.532Z 2 040-059 66 66 58 128 128 112 F F F <---
72042024-01-10T17:58:07.532Z 3 060-079 66 66 62 128 128 121 F F F <---
72052024-01-10T17:58:07.532Z 4 080-099 58 58 58 109 109 109 F F F
72062024-01-10T17:58:07.536Z 5 100-119 64 64 64 126 126 126 F F F
72072024-01-10T17:58:07.536Z 6 120-139 60 60 60 117 117 117 F F F
72082024-01-10T17:58:07.536Z 7 140-159 65 65 65 127 127 127 F F F
72092024-01-10T17:58:07.536Z 8 160-179 64 64 64 125 125 125 F F F
72102024-01-10T17:58:07.536Z 9 180-199 65 65 65 127 127 127 T T F <---
72112024-01-10T17:58:07.536Z 10 200-219 62 62 62 123 123 123 T T F <---
72122024-01-10T17:58:07.536Z 11 220-239 60 60 60 117 117 117 T T F <---
72132024-01-10T17:58:07.536Z 12 240-259 62 62 62 121 121 121 T T F <---
72142024-01-10T17:58:07.537Z 13 260-279 65 65 65 127 127 127 F F F
72152024-01-10T17:58:07.537Z 14 280-299 65 65 65 127 127 127 T T F <---
72162024-01-10T17:58:07.537Z 15 300-319 64 64 64 126 126 126 F F F
72172024-01-10T17:58:07.537Z 16 320-339 64 64 64 126 126 126 F F F
72182024-01-10T17:58:07.537Z 17 340-359 62 62 62 121 121 121 F F F
72192024-01-10T17:58:07.537Z 18 360-379 58 58 58 111 111 111 F F F
72202024-01-10T17:58:07.537Z 19 380-399 58 58 58 112 112 112 F F F
72212024-01-10T17:58:07.537Z 20 400-419 65 65 65 127 127 127 F F F
72222024-01-10T17:58:07.537Z 21 420-439 59 59 59 113 113 113 F F F
72232024-01-10T17:58:07.537Z 22 440-459 65 65 65 127 127 127 T T F <---
72242024-01-10T17:58:07.537Z 23 460-479 63 63 63 124 124 124 F F F
72252024-01-10T17:58:07.537Z 24 480-499 58 58 58 109 109 109 F F F
72262024-01-10T17:58:07.537Z 25 500-519 60 60 60 117 117 117 F F F
72272024-01-10T17:58:07.538Z 26 520-539 66 66 60 128 128 116 F F F <---
72282024-01-10T17:58:07.538Z 27 540-559 64 64 64 126 126 126 F F F
72292024-01-10T17:58:07.538Z 28 560-579 64 64 64 126 126 126 F F F
72302024-01-10T17:58:07.538Z 29 580-599 64 64 64 126 126 126 F F F
72312024-01-10T17:58:07.538ZMax gen: 66, Max flush: 128
72322024-01-10T17:58:07.538ZError: Difference in extent metadata found!
72332024-01-10T17:58:07.538ZA Difference in extent metadata is expected here
72342024-01-10T17:58:07.538ZOn loop 33
72352024-01-10T17:58:07.538Z
72362024-01-10T17:58:07.542ZVerifying data now
72372024-01-10T17:58:12.657ZNo change after verify
72382024-01-10T17:58:12.657ZLoop: 33 Downstairs dump after verify (and repair):
72392024-01-10T17:58:12.679ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72402024-01-10T17:58:12.679Z 0 000-019 67 67 67 129 129 129 F F F
72412024-01-10T17:58:12.679Z 1 020-039 66 66 66 128 128 128 F F F
72422024-01-10T17:58:12.679Z 2 040-059 66 66 66 128 128 128 F F F
72432024-01-10T17:58:12.679Z 3 060-079 66 66 66 128 128 128 F F F
72442024-01-10T17:58:12.680Z 4 080-099 58 58 58 109 109 109 F F F
72452024-01-10T17:58:12.680Z 5 100-119 64 64 64 126 126 126 F F F
72462024-01-10T17:58:12.684Z 6 120-139 60 60 60 117 117 117 F F F
72472024-01-10T17:58:12.684Z 7 140-159 65 65 65 127 127 127 F F F
72482024-01-10T17:58:12.684Z 8 160-179 64 64 64 125 125 125 F F F
72492024-01-10T17:58:12.684Z 9 180-199 67 67 67 129 129 129 F F F
72502024-01-10T17:58:12.684Z 10 200-219 67 67 67 129 129 129 F F F
72512024-01-10T17:58:12.684Z 11 220-239 67 67 67 129 129 129 F F F
72522024-01-10T17:58:12.684Z 12 240-259 67 67 67 129 129 129 F F F
72532024-01-10T17:58:12.684Z 13 260-279 65 65 65 127 127 127 F F F
72542024-01-10T17:58:12.684Z 14 280-299 67 67 67 129 129 129 F F F
72552024-01-10T17:58:12.684Z 15 300-319 64 64 64 126 126 126 F F F
72562024-01-10T17:58:12.684Z 16 320-339 64 64 64 126 126 126 F F F
72572024-01-10T17:58:12.684Z 17 340-359 62 62 62 121 121 121 F F F
72582024-01-10T17:58:12.684Z 18 360-379 58 58 58 111 111 111 F F F
72592024-01-10T17:58:12.684Z 19 380-399 58 58 58 112 112 112 F F F
72602024-01-10T17:58:12.684Z 20 400-419 65 65 65 127 127 127 F F F
72612024-01-10T17:58:12.684Z 21 420-439 59 59 59 113 113 113 F F F
72622024-01-10T17:58:12.684Z 22 440-459 67 67 67 129 129 129 F F F
72632024-01-10T17:58:12.685Z 23 460-479 63 63 63 124 124 124 F F F
72642024-01-10T17:58:12.685Z 24 480-499 58 58 58 109 109 109 F F F
72652024-01-10T17:58:12.685Z 25 500-519 60 60 60 117 117 117 F F F
72662024-01-10T17:58:12.685Z 26 520-539 66 66 66 128 128 128 F F F
72672024-01-10T17:58:12.685Z 27 540-559 64 64 64 126 126 126 F F F
72682024-01-10T17:58:12.685Z 28 560-579 64 64 64 126 126 126 F F F
72692024-01-10T17:58:12.685Z 29 580-599 64 64 64 126 126 126 F F F
72702024-01-10T17:58:12.685ZMax gen: 67, Max flush: 129
72712024-01-10T17:58:12.685Z
72722024-01-10T17:58:12.685ZBegin loop 34
72732024-01-10T17:58:12.685ZDownstairs to restart: 2
72742024-01-10T17:58:12.713Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.714753367Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427}
72752024-01-10T17:58:12.713Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.715233495Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427}
72762024-01-10T17:58:12.717Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.715267728Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427}
72772024-01-10T17:58:12.717Z{"msg":"Crucible 3ef472b1-9a83-4a9b-b3e3-a595fff55d57 has session id: fab68052-428a-4e20-8790-9770eae0c3bb","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.715337869Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72782024-01-10T17:58:12.719Z{"msg":"Upstairs opts: Upstairs UUID: 3ef472b1-9a83-4a9b-b3e3-a595fff55d57, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.715513875Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72792024-01-10T17:58:12.719Z{"msg":"Crucible stats registered with UUID: 3ef472b1-9a83-4a9b-b3e3-a595fff55d57","v":0,"name":"crucible","level":30Crucible runtime is spawned
72802024-01-10T17:58:12.719Z,"time":"2024-01-10T17:58:12.715554363Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72812024-01-10T17:58:12.719Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.715705169Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427}
72822024-01-10T17:58:12.719Z{"msg":"3ef472b1-9a83-4a9b-b3e3-a595fff55d57 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.71592141Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72832024-01-10T17:58:12.719Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.715987925Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"0","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72842024-01-10T17:58:12.719Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.716032294Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"0","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72852024-01-10T17:58:12.719Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.716086186Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"1","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72862024-01-10T17:58:12.719Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.716128779Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"1","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72872024-01-10T17:58:12.719Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.716169176Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"2","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72882024-01-10T17:58:12.719Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.716225508Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"2","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72892024-01-10T17:58:12.719Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.71626814Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72902024-01-10T17:58:12.719Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.716359969Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72912024-01-10T17:58:12.720Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.716406096Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72922024-01-10T17:58:12.720Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.716592909Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72932024-01-10T17:58:12.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.716625085Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72942024-01-10T17:58:12.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.716665888Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72952024-01-10T17:58:12.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.718058273Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"0","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72962024-01-10T17:58:12.721Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.718146576Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"0","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72972024-01-10T17:58:12.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.718247953Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"1","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72982024-01-10T17:58:12.721Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.718304835Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"1","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
72992024-01-10T17:58:12.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.718565695Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"2","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73002024-01-10T17:58:12.721Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.718658926Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"2","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73012024-01-10T17:58:12.725Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.719134158Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"0","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73022024-01-10T17:58:12.726Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.719208141Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"1","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73032024-01-10T17:58:12.726Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.71975732Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"2","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73042024-01-10T17:58:12.726Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720066017Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"1","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73052024-01-10T17:58:12.726Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720136634Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"1","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73062024-01-10T17:58:12.726Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720161237Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73072024-01-10T17:58:12.726Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720298773Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"0","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73082024-01-10T17:58:12.726Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720350845Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"0","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73092024-01-10T17:58:12.726Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720375423Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73102024-01-10T17:58:12.726Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720628045Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"2","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73112024-01-10T17:58:12.726Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720737236Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"2","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73122024-01-10T17:58:12.726Z{"msg":"[0]R flush_numbers[0..12]: [129, 128, 128, 128, 109, 126, 117, 127, 125, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.7207851Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73132024-01-10T17:58:12.726Z{"msg":"[0]R generation[0..12]: [67, 66, 66, 66, 58, 64, 60, 65, 64, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720829859Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73142024-01-10T17:58:12.727Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720874643Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
73152024-01-10T17:58:12.727Z,"":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73162024-01-10T17:58:12.727Z{"msg":"[1]R flush_numbers[0..12]: [129, 128, 128, 128, 109, 126, 117, 127, 125, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720922995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73172024-01-10T17:58:12.727Z{"msg":"[1]R generation[0..12]: [67, 66, 66, 66, 58, 64, 60, 65, 64, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.720992367Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73182024-01-10T17:58:12.727Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721037751Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73192024-01-10T17:58:12.727Z{"msg":"[2]R flush_numbers[0..12]: [129, 128, 128, 128, 109, 126, 117, 127, 125, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721093636Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73202024-01-10T17:58:12.727Z{"msg":"[2]R generation[0..12]: [67, 66, 66, 66, 58, 64, 60, 65, 64, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721192772Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73212024-01-10T17:58:12.727Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721231102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73222024-01-10T17:58:12.728Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721273223Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73232024-01-10T17:58:12.728Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721296863Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73242024-01-10T17:58:12.728Z{"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721318092Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73252024-01-10T17:58:12.728Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721356966Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"":"downstairs","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73262024-01-10T17:58:12.728Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721379725Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73272024-01-10T17:58:12.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721436567Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"0","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73282024-01-10T17:58:12.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.72146217Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73292024-01-10T17:58:12.729Zpid":1427,"client":"1","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73302024-01-10T17:58:12.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721546059Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"client":"2","":"downstairs","session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73312024-01-10T17:58:12.729Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721593529Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73322024-01-10T17:58:12.729Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721616093Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73332024-01-10T17:58:12.729Z{"msg":"3ef472b1-9a83-4a9b-b3e3-a595fff55d57 is now active with session: fab68052-428a-4e20-8790-9770eae0c3bb","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721655272Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73342024-01-10T17:58:12.729Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721675923Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427,"session_id":"fab68052-428a-4e20-8790-9770eae0c3bb"}
73352024-01-10T17:58:12.729Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:12.721713477Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1427}
73362024-01-10T17:58:12.729ZLoading write count information from file "/tmp/test_repair_verify.data"
73372024-01-10T17:58:12.729ZRun Repair workload
73382024-01-10T17:58:12.729Z01/30 Write block 420 len 8192 data: 10 9
73392024-01-10T17:58:12.729Z02/30 Read block 441 len 16384
73402024-01-10T17:58:12.733Z03/30 Read block 499 len 36864
73412024-01-10T17:58:12.733Z04/30 Write block 336 len 20480 data: 6 7 6 6 5
73422024-01-10T17:58:12.733Z05/30 Write block 318 len 24576 data: 5 6 6 6 6 6
73432024-01-10T17:58:12.733Z06/30 Read block 134 len 32768
73442024-01-10T17:58:12.737Z07/30 Write block 61 len 32768 data: 4 4 4 7 8 9 6 5
73452024-01-10T17:58:12.737Z08/30 Read block 128 len 36864
73462024-01-10T17:58:12.740Z09/30 Write block 35 len 20480 data: 6 6 5 6 4
73472024-01-10T17:58:12.740Z10/30 Read block 533 len 4096
73482024-01-10T17:58:12.743Z11/30 Write block 150 len 20480 data: 5 4 2 2 2
73492024-01-10T17:58:12.743Z12/30 Read block 147 len 4096
73502024-01-10T17:58:12.743Z13/30 Read block 434 len 28672
73512024-01-10T17:58:12.747Z14/30 Read block 278 len 40960
73522024-01-10T17:58:12.749Z15/30 Read block 156 len 40960
73532024-01-10T17:58:12.752Z16/30 Write block 417 len 28672 data: 5 6 10 11 10 9 9
73542024-01-10T17:58:12.756Z17/30 Write block 496 len 36864 data: 12 10 10 10 7 7 9 10 8
73552024-01-10T17:58:12.756Z18/30 Write block 58 len 16384 data: 6 6 6 5
73562024-01-10T17:58:12.759Z19/30 Read block 181 len 40960
73572024-01-10T17:58:12.762Z20/30 Read block 469 len 20480
73582024-01-10T17:58:12.765Z21/30 Flush
73592024-01-10T17:58:12.768Z22/30 Write block 75 len 12288 data: 6 6 5
73602024-01-10T17:58:12.768Z23/30 Write block 28 len 40960 data: 6 7 10 8 8 7 6 7 7 6
73612024-01-10T17:58:12.771Z24/30 Read block 394 len 36864
73622024-01-10T17:58:12.774Z25/30 Read block 377 len 36864
73632024-01-10T17:58:12.778Z26/30 Write block 287 len 16384 data: 5 5 7 10
73642024-01-10T17:58:12.778Z27/30 Write block 147 len 24576 data: 6 5 5 6 5 3
73652024-01-10T17:58:12.780Z28/30 Read block 411 len 16384
73662024-01-10T17:58:12.780Z29/30 Read block 334 len 28672
73672024-01-10T17:58:12.784Z30/30 Read block 436 len 28672
73682024-01-10T17:58:12.787Z----------------------------------------------------------------
73692024-01-10T17:58:12.787Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
73702024-01-10T17:58:12.787ZStates: Active Active Active
73712024-01-10T17:58:12.787ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
73722024-01-10T17:58:12.788Z 1 Acked 1000 Write 2 Done Done Sent false
73732024-01-10T17:58:12.788Z 2 Acked 1001 Read 4 Done Done Sent false
73742024-01-10T17:58:12.788Z 3 Acked 1002 Read 9 Done Done Sent false
73752024-01-10T17:58:12.788Z 4 Acked 1003 Write 5 Done Done Sent false
73762024-01-10T17:58:12.788Z 5 Acked 1004 Write 6 Done Done Sent false
73772024-01-10T17:58:12.788Z 6 Acked 1005 Read 8 Done Done Sent false
73782024-01-10T17:58:12.788Z 7 Acked 1006 Write 8 Done Done Sent false
73792024-01-10T17:58:12.789Z 8 Acked 1007 Read 9 Done Done Sent false
73802024-01-10T17:58:12.789Z 9 Acked 1008 Write 5 Done Done Sent false
73812024-01-10T17:58:12.790Z 10 Acked 1009 Read 1 Done Done Sent false
73822024-01-10T17:58:12.790Z 11 Acked 1010 Write 5 Done Done Sent false
73832024-01-10T17:58:12.790Z 12 Acked 1011 Read 1 Done Done Sent false
73842024-01-10T17:58:12.790Z 13 Acked 1012 Read 7 Done Done Sent false
73852024-01-10T17:58:12.790Z 14 Acked 1013 Read 10 Done Done Sent false
73862024-01-10T17:58:12.790Z 15 Acked 1014 Read 10 Done Done Sent false
73872024-01-10T17:58:12.790Z 16 Acked 1015 Write 7 Done Done Sent false
73882024-01-10T17:58:12.790Z 17 Acked 1016 Write 9 Done Done Sent false
73892024-01-10T17:58:12.790Z 18 Acked 1017 Write 4 Done Done Sent false
73902024-01-10T17:58:12.790Z 19 Acked 1018 Read 10 Done Done Sent false
73912024-01-10T17:58:12.790Z 20 Acked 1019 Read 5 Done Done Sent false
73922024-01-10T17:58:12.790Z 21 Acked 1020 Flush 0 Done Done Sent false
73932024-01-10T17:58:12.790Z 22 Acked 1021 Write 3 Done Done Sent false
73942024-01-10T17:58:12.790Z 23 Acked 1022 Write 10 Done Done Sent false
73952024-01-10T17:58:12.790Z 24 Acked 1023 Read 9 Done Done Sent false
73962024-01-10T17:58:12.790Z 25 Acked 1024 Read 9 Done Done Sent false
73972024-01-10T17:58:12.790Z 26 Acked 1025 Write 4 Done Done Sent false
73982024-01-10T17:58:12.790Z 27 Acked 1026 Write 6 Done Done Sent false
73992024-01-10T17:58:12.790Z 28 Acked 1027 Read 4 Done Done Sent false
74002024-01-10T17:58:12.790Z 29 Acked 1028 Read 7 Done Done Sent false
74012024-01-10T17:58:12.790Z 30 Acked 1029 Read 7 Done Done Sent false
74022024-01-10T17:58:12.791Z STATES DS:0 DS:1 DS:2 TOTAL
74032024-01-10T17:58:12.791Z New 0 0 0 0
74042024-01-10T17:58:12.791Z Sent 0 0 30 30
74052024-01-10T17:58:12.791Z Done 30 30 0 60
74062024-01-10T17:58:12.791Z Skipped 0 0 0 0
74072024-01-10T17:58:12.791Z Error 0 0 0 0
74082024-01-10T17:58:12.791ZLast Flush: 1020 1020 0
74092024-01-10T17:58:12.791ZDownstairs last five completed:
74102024-01-10T17:58:12.791ZUpstairs last five completed: 30 29 28 27 26
74112024-01-10T17:58:12.792ZWrote out file "/tmp/test_repair_verify.data"
74122024-01-10T17:58:12.795Z
74132024-01-10T17:58:12.799ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74142024-01-10T17:58:12.825ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74152024-01-10T17:58:12.825Z 0 000-019 67 67 67 129 129 129 F F F
74162024-01-10T17:58:12.825Z 1 020-039 68 68 66 130 130 128 T T F <---
74172024-01-10T17:58:12.825Z 2 040-059 68 68 66 130 130 128 F F F <---
74182024-01-10T17:58:12.825Z 3 060-079 68 68 66 130 130 128 T T F <---
74192024-01-10T17:58:12.825Z 4 080-099 58 58 58 109 109 109 F F F
74202024-01-10T17:58:12.825Z 5 100-119 64 64 64 126 126 126 F F F
74212024-01-10T17:58:12.829Z 6 120-139 60 60 60 117 117 117 F F F
74222024-01-10T17:58:12.829Z 7 140-159 68 68 65 130 130 127 T T F <---
74232024-01-10T17:58:12.829Z 8 160-179 64 64 64 125 125 125 F F F
74242024-01-10T17:58:12.829Z 9 180-199 67 67 67 129 129 129 F F F
74252024-01-10T17:58:12.829Z 10 200-219 67 67 67 129 129 129 F F F
74262024-01-10T17:58:12.829Z 11 220-239 67 67 67 129 129 129 F F F
74272024-01-10T17:58:12.831Z 12 240-259 67 67 67 129 129 129 F F F
74282024-01-10T17:58:12.831Z 13 260-279 65 65 65 127 127 127 F F F
74292024-01-10T17:58:12.831Z 14 280-299 67 67 67 129 129 129 T T F <---
74302024-01-10T17:58:12.832Z 15 300-319 68 68 64 130 130 126 F F F <---
74312024-01-10T17:58:12.832Z 16 320-339 68 68 64 130 130 126 F F F <---
74322024-01-10T17:58:12.832Z 17 340-359 68 68 62 130 130 121 F F F <---
74332024-01-10T17:58:12.832Z 18 360-379 58 58 58 111 111 111 F F F
74342024-01-10T17:58:12.832Z 19 380-399 58 58 58 112 112 112 F F F
74352024-01-10T17:58:12.832Z 20 400-419 68 68 65 130 130 127 F F F <---
74362024-01-10T17:58:12.832Z 21 420-439 68 68 59 130 130 113 F F F <---
74372024-01-10T17:58:12.832Z 22 440-459 67 67 67 129 129 129 F F F
74382024-01-10T17:58:12.832Z 23 460-479 63 63 63 124 124 124 F F F
74392024-01-10T17:58:12.832Z 24 480-499 68 68 58 130 130 109 F F F <---
74402024-01-10T17:58:12.832Z 25 500-519 68 68 60 130 130 117 F F F <---
74412024-01-10T17:58:12.832Z 26 520-539 66 66 66 128 128 128 F F F
74422024-01-10T17:58:12.832Z 27 540-559 64 64 64 126 126 126 F F F
74432024-01-10T17:58:12.832Z 28 560-579 64 64 64 126 126 126 F F F
74442024-01-10T17:58:12.832Z 29 580-599 64 64 64 126 126 126 F F F
74452024-01-10T17:58:12.832ZMax gen: 68, Max flush: 130
74462024-01-10T17:58:12.832ZError: Difference in extent metadata found!
74472024-01-10T17:58:12.836ZA Difference in extent metadata is expected here
74482024-01-10T17:58:12.836ZOn loop 34
74492024-01-10T17:58:12.836Z
74502024-01-10T17:58:12.836ZVerifying data now
74512024-01-10T17:58:18.033ZNo change after verify
74522024-01-10T17:58:18.033ZLoop: 34 Downstairs dump after verify (and repair):
74532024-01-10T17:58:18.052ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74542024-01-10T17:58:18.052Z 0 000-019 67 67 67 129 129 129 F F F
74552024-01-10T17:58:18.052Z 1 020-039 69 69 69 131 131 131 F F F
74562024-01-10T17:58:18.052Z 2 040-059 68 68 68 130 130 130 F F F
74572024-01-10T17:58:18.052Z 3 060-079 69 69 69 131 131 131 F F F
74582024-01-10T17:58:18.052Z 4 080-099 58 58 58 109 109 109 F F F
74592024-01-10T17:58:18.052Z 5 100-119 64 64 64 126 126 126 F F F
74602024-01-10T17:58:18.056Z 6 120-139 60 60 60 117 117 117 F F F
74612024-01-10T17:58:18.056Z 7 140-159 69 69 69 131 131 131 F F F
74622024-01-10T17:58:18.056Z 8 160-179 64 64 64 125 125 125 F F F
74632024-01-10T17:58:18.056Z 9 180-199 67 67 67 129 129 129 F F F
74642024-01-10T17:58:18.057Z 10 200-219 67 67 67 129 129 129 F F F
74652024-01-10T17:58:18.057Z 11 220-239 67 67 67 129 129 129 F F F
74662024-01-10T17:58:18.057Z 12 240-259 67 67 67 129 129 129 F F F
74672024-01-10T17:58:18.057Z 13 260-279 65 65 65 127 127 127 F F F
74682024-01-10T17:58:18.057Z 14 280-299 69 69 69 131 131 131 F F F
74692024-01-10T17:58:18.057Z 15 300-319 68 68 68 130 130 130 F F F
74702024-01-10T17:58:18.057Z 16 320-339 68 68 68 130 130 130 F F F
74712024-01-10T17:58:18.057Z 17 340-359 68 68 68 130 130 130 F F F
74722024-01-10T17:58:18.058Z 18 360-379 58 58 58 111 111 111 F F F
74732024-01-10T17:58:18.058Z 19 380-399 58 58 58 112 112 112 F F F
74742024-01-10T17:58:18.058Z 20 400-419 68 68 68 130 130 130 F F F
74752024-01-10T17:58:18.058Z 21 420-439 68 68 68 130 130 130 F F F
74762024-01-10T17:58:18.058Z 22 440-459 67 67 67 129 129 129 F F F
74772024-01-10T17:58:18.058Z 23 460-479 63 63 63 124 124 124 F F F
74782024-01-10T17:58:18.058Z 24 480-499 68 68 68 130 130 130 F F F
74792024-01-10T17:58:18.058Z 25 500-519 68 68 68 130 130 130 F F F
74802024-01-10T17:58:18.058Z 26 520-539 66 66 66 128 128 128 F F F
74812024-01-10T17:58:18.058Z 27 540-559 64 64 64 126 126 126 F F F
74822024-01-10T17:58:18.058Z 28 560-579 64 64 64 126 126 126 F F F
74832024-01-10T17:58:18.058Z 29 580-599 64 64 64 126 126 126 F F F
74842024-01-10T17:58:18.058ZMax gen: 69, Max flush: 131
74852024-01-10T17:58:18.061Z
74862024-01-10T17:58:18.061ZBegin loop 35
74872024-01-10T17:58:18.061ZDownstairs to restart: 1
74882024-01-10T17:58:18.086Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.088043711Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435}
74892024-01-10T17:58:18.090Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.088591933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435}
74902024-01-10T17:58:18.090Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.08874381Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435}
74912024-01-10T17:58:18.090ZCrucible runtime is spawned
74922024-01-10T17:58:18.090Z{"msg":"Crucible 345d4479-d68e-482f-85d5-9ca3ef51f7c9 has session id: 5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.088893692Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
74932024-01-10T17:58:18.090Z{"msg":"Upstairs opts: Upstairs UUID: 345d4479-d68e-482f-85d5-9ca3ef51f7c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.088980942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
74942024-01-10T17:58:18.090Z{"msg":"Crucible stats registered with UUID: 345d4479-d68e-482f-85d5-9ca3ef51f7c9","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089010629Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
74952024-01-10T17:58:18.090Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.08913492Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435}
74962024-01-10T17:58:18.090Z{"msg":"345d4479-d68e-482f-85d5-9ca3ef51f7c9 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.08929474Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
74972024-01-10T17:58:18.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089365305Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"0","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
74982024-01-10T17:58:18.091Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089405572Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"0","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
74992024-01-10T17:58:18.092Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089430837Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"1","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75002024-01-10T17:58:18.092Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089467651Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"1","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75012024-01-10T17:58:18.092Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089518036Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"2","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75022024-01-10T17:58:18.092Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089542806Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"2","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75032024-01-10T17:58:18.092Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089569456Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75042024-01-10T17:58:18.092Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089735174Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75052024-01-10T17:58:18.092Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089833215Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75062024-01-10T17:58:18.092Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.08990424Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75072024-01-10T17:58:18.092Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.089972782Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75082024-01-10T17:58:18.092Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.090154402Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75092024-01-10T17:58:18.093Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.091288964Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"1","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75102024-01-10T17:58:18.093Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.09141623Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"1","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75112024-01-10T17:58:18.093Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.091491575Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"0","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75122024-01-10T17:58:18.093Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.091536284Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"0","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75132024-01-10T17:58:18.093Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.09162561Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"2","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75142024-01-10T17:58:18.097Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.091669694Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"2","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75152024-01-10T17:58:18.097Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.09246304Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"2","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75162024-01-10T17:58:18.097Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.092587942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"0","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75172024-01-10T17:58:18.097Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.0927101Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"1","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75182024-01-10T17:58:18.099Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.093545248Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"2","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75192024-01-10T17:58:18.099Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.093635468Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"2","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75202024-01-10T17:58:18.099Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.093710798Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75212024-01-10T17:58:18.099Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.093776962Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"0","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75222024-01-10T17:58:18.099Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.093847551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"0","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75232024-01-10T17:58:18.099Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.093917144Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75242024-01-10T17:58:18.100Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.093982746Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
75252024-01-10T17:58:18.100Z,"pid":1435,"client":"1","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75262024-01-10T17:58:18.100Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094032257Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"1","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75272024-01-10T17:58:18.100Z{"msg":"[0]R flush_numbers[0..12]: [129, 131, 130, 131, 109, 126, 117, 131, 125, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.09410889Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75282024-01-10T17:58:18.100Z{"msg":"[0]R generation[0..12]: [67, 69, 68, 69, 58, 64, 60, 69, 64, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094187053Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75292024-01-10T17:58:18.108Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094265825Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75302024-01-10T17:58:18.108Z{"msg":"[1]R flush_numbers[0..12]: [129, 131, 130, 131, 109, 126, 117, 131, 125, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094344016Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75312024-01-10T17:58:18.108Z{"msg":"[1]R generation[0..12]: [67, 69, 68, 69, 58, 64, 60, 69, 64, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094419724Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75322024-01-10T17:58:18.108Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094495994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75332024-01-10T17:58:18.108Z{"msg":"[2]R flush_numbers[0..12]: [129, 131, 130, 131, 109, 126, 117, 131, 125, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.09457243Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75342024-01-10T17:58:18.108ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
75352024-01-10T17:58:18.108Z{"msg":"[2]R generation[0..12]: [67, 69, 68, 69, 58, 64, 60, 69, 64, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094648692Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75362024-01-10T17:58:18.108Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094689727Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75372024-01-10T17:58:18.108Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094734374Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75382024-01-10T17:58:18.108Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094758423Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75392024-01-10T17:58:18.110Z{"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094779524Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75402024-01-10T17:58:18.110Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094800611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"":"downstairs","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75412024-01-10T17:58:18.110Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094839835Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75422024-01-10T17:58:18.110Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094860437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"0","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75432024-01-10T17:58:18.110Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094881788Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"1","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75442024-01-10T17:58:18.110Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094902951Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"client":"2","":"downstairs","session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75452024-01-10T17:58:18.110Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094924432Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75462024-01-10T17:58:18.110Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094962017Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75472024-01-10T17:58:18.110Z{"msg":"345d4479-d68e-482f-85d5-9ca3ef51f7c9 is now active with session: 5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.094984588Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75482024-01-10T17:58:18.110Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.095003548Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435,"session_id":"5a3bd8e8-a50c-4e1f-a14a-baad7dad9a8a"}
75492024-01-10T17:58:18.110Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:18.095041097Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1435}
75502024-01-10T17:58:18.110ZLoading write count information from file "/tmp/test_repair_verify.data"
75512024-01-10T17:58:18.110ZRun Repair workload
75522024-01-10T17:58:18.110Z01/30 Write block 279 len 36864 data: 8 7 8 7 8 7 7 5 6
75532024-01-10T17:58:18.114Z02/30 Write block 516 len 28672 data: 8 8 7 6 6 6 4
75542024-01-10T17:58:18.114Z03/30 Write block 390 len 40960 data: 3 4 4 3 4 4 4 4 5 7
75552024-01-10T17:58:18.114Z04/30 Read block 317 len 16384
75562024-01-10T17:58:18.114Z05/30 Write block 67 len 16384 data: 7 6 5 6
75572024-01-10T17:58:18.114Z06/30 Write block 155 len 12288 data: 3 3 3
75582024-01-10T17:58:18.114Z07/30 Write block 59 len 20480 data: 7 7 6 5 5
75592024-01-10T17:58:18.114Z08/30 Read block 547 len 12288
75602024-01-10T17:58:18.114Z09/30 Read block 251 len 4096
75612024-01-10T17:58:18.115Z10/30 Write block 323 len 20480 data: 7 8 8 6 6
75622024-01-10T17:58:18.115Z11/30 Flush
75632024-01-10T17:58:18.118Z12/30 Write block 593 len 24576 data: 7 8 7 6 5 4
75642024-01-10T17:58:18.118Z13/30 Write block 88 len 24576 data: 3 3 3 6 5 5
75652024-01-10T17:58:18.118Z14/30 Write block 511 len 20480 data: 9 10 10 11 9
75662024-01-10T17:58:18.118Z15/30 Read block 384 len 24576
75672024-01-10T17:58:18.121Z16/30 Write block 266 len 40960 data: 5 7 6 7 7 7 5 6 9 10
75682024-01-10T17:58:18.124Z17/30 Write block 202 len 8192 data: 7 6
75692024-01-10T17:58:18.124Z18/30 Read block 399 len 32768
75702024-01-10T17:58:18.127Z19/30 Write block 583 len 24576 data: 7 5 4 6 5 5
75712024-01-10T17:58:18.130Z20/30 Write block 529 len 28672 data: 8 9 11 9 9 8 7
75722024-01-10T17:58:18.130Z21/30 Read block 310 len 28672
75732024-01-10T17:58:18.133Z22/30 Flush
75742024-01-10T17:58:18.137Z23/30 Flush
75752024-01-10T17:58:18.137Z24/30 Write block 148 len 36864 data: 6 6 7 6 4 3 3 4 4
75762024-01-10T17:58:18.137Z25/30 Read block 17 len 12288
75772024-01-10T17:58:18.140Z26/30 Read block 166 len 36864
75782024-01-10T17:58:18.146Z27/30 Read block 84 len 24576
75792024-01-10T17:58:18.146Z28/30 Read block 402 len 24576
75802024-01-10T17:58:18.149Z29/30 Write block 154 len 8192 data: 4 5
75812024-01-10T17:58:18.149Z30/30 Read block 199 len 20480
75822024-01-10T17:58:18.155Z----------------------------------------------------------------
75832024-01-10T17:58:18.155Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
75842024-01-10T17:58:18.155ZStates: Active Active Active
75852024-01-10T17:58:18.156ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
75862024-01-10T17:58:18.156Z 1 Acked 1000 Write 9 Done Sent Done false
75872024-01-10T17:58:18.156Z 2 Acked 1001 Write 7 Done Sent Done false
75882024-01-10T17:58:18.156Z 3 Acked 1002 Write 10 Done Sent Done false
75892024-01-10T17:58:18.156Z 4 Acked 1003 Read 4 Done Sent Done false
75902024-01-10T17:58:18.156Z 5 Acked 1004 Write 4 Done Sent Done false
75912024-01-10T17:58:18.156Z 6 Acked 1005 Write 3 Done Sent Done false
75922024-01-10T17:58:18.156Z 7 Acked 1006 Write 5 Done Sent Done false
75932024-01-10T17:58:18.156Z 8 Acked 1007 Read 3 Done Sent Done false
75942024-01-10T17:58:18.156Z 9 Acked 1008 Read 1 Done Sent Done false
75952024-01-10T17:58:18.156Z 10 Acked 1009 Write 5 Done Sent Done false
75962024-01-10T17:58:18.156Z 11 Acked 1010 Flush 0 Done Sent Done false
75972024-01-10T17:58:18.156Z 12 Acked 1011 Write 6 Done Sent Done false
75982024-01-10T17:58:18.156Z 13 Acked 1012 Write 6 Done Sent Done false
75992024-01-10T17:58:18.157Z 14 Acked 1013 Write 5 Done Sent Done false
76002024-01-10T17:58:18.157Z 15 Acked 1014 Read 6 Done Sent Done false
76012024-01-10T17:58:18.157Z 16 Acked 1015 Write 10 Done Sent Done false
76022024-01-10T17:58:18.157Z 17 Acked 1016 Write 2 Done Sent Done false
76032024-01-10T17:58:18.157Z 18 Acked 1017 Read 8 Done Sent Done false
76042024-01-10T17:58:18.157Z 19 Acked 1018 Write 6 Done Sent Done false
76052024-01-10T17:58:18.157Z 20 Acked 1019 Write 7 Done Sent Done false
76062024-01-10T17:58:18.157Z 21 Acked 1020 Read 7 Done Sent Done false
76072024-01-10T17:58:18.157Z 22 Acked 1021 Flush 0 Done Sent Done false
76082024-01-10T17:58:18.157Z 23 Acked 1022 Flush 0 Done Sent Done false
76092024-01-10T17:58:18.157Z 24 Acked 1023 Write 9 Done Sent Done false
76102024-01-10T17:58:18.157Z 25 Acked 1024 Read 3 Done Sent Done false
76112024-01-10T17:58:18.157Z 26 Acked 1025 Read 9 Done Sent Done false
76122024-01-10T17:58:18.157Z 27 Acked 1026 Read 6 Done Sent Done false
76132024-01-10T17:58:18.157Z 28 Acked 1027 Read 6 Done Sent Done false
76142024-01-10T17:58:18.157Z 29 Acked 1028 Write 2 Done Sent Done false
76152024-01-10T17:58:18.158Z 30 Acked 1029 Read 5 Done Sent Done false
76162024-01-10T17:58:18.158Z STATES DS:0 DS:1 DS:2 TOTAL
76172024-01-10T17:58:18.158Z New 0 0 0 0
76182024-01-10T17:58:18.158Z Sent 0 30 0 30
76192024-01-10T17:58:18.158Z Done 30 0 30 60
76202024-01-10T17:58:18.158Z Skipped 0 0 0 0
76212024-01-10T17:58:18.158Z Error 0 0 0 0
76222024-01-10T17:58:18.158ZLast Flush: 1022 0 1022
76232024-01-10T17:58:18.159ZDownstairs last five completed:
76242024-01-10T17:58:18.159ZUpstairs last five completed: 30 29 28 27 26
76252024-01-10T17:58:18.159ZWrote out file "/tmp/test_repair_verify.data"
76262024-01-10T17:58:18.162Z
76272024-01-10T17:58:18.165ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76282024-01-10T17:58:18.185ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76292024-01-10T17:58:18.185Z 0 000-019 67 67 67 129 129 129 F F F
76302024-01-10T17:58:18.185Z 1 020-039 69 69 69 131 131 131 F F F
76312024-01-10T17:58:18.185Z 2 040-059 70 68 70 132 130 132 F F F <---
76322024-01-10T17:58:18.185Z 3 060-079 70 69 70 132 131 132 F F F <---
76332024-01-10T17:58:18.185Z 4 080-099 70 58 70 133 109 133 F F F <---
76342024-01-10T17:58:18.188Z 5 100-119 64 64 64 126 126 126 F F F
76352024-01-10T17:58:18.188Z 6 120-139 60 60 60 117 117 117 F F F
76362024-01-10T17:58:18.188Z 7 140-159 70 69 70 132 131 132 T F T <---
76372024-01-10T17:58:18.188Z 8 160-179 64 64 64 125 125 125 F F F
76382024-01-10T17:58:18.188Z 9 180-199 67 67 67 129 129 129 F F F
76392024-01-10T17:58:18.188Z 10 200-219 70 67 70 133 129 133 F F F <---
76402024-01-10T17:58:18.189Z 11 220-239 67 67 67 129 129 129 F F F
76412024-01-10T17:58:18.189Z 12 240-259 67 67 67 129 129 129 F F F
76422024-01-10T17:58:18.189Z 13 260-279 70 65 70 133 127 133 F F F <---
76432024-01-10T17:58:18.189Z 14 280-299 70 69 70 132 131 132 F F F <---
76442024-01-10T17:58:18.189Z 15 300-319 68 68 68 130 130 130 F F F
76452024-01-10T17:58:18.189Z 16 320-339 70 68 70 132 130 132 F F F <---
76462024-01-10T17:58:18.189Z 17 340-359 68 68 68 130 130 130 F F F
76472024-01-10T17:58:18.189Z 18 360-379 58 58 58 111 111 111 F F F
76482024-01-10T17:58:18.189Z 19 380-399 70 58 70 132 112 132 F F F <---
76492024-01-10T17:58:18.190Z 20 400-419 68 68 68 130 130 130 F F F
76502024-01-10T17:58:18.190Z 21 420-439 68 68 68 130 130 130 F F F
76512024-01-10T17:58:18.190Z 22 440-459 67 67 67 129 129 129 F F F
76522024-01-10T17:58:18.190Z 23 460-479 63 63 63 124 124 124 F F F
76532024-01-10T17:58:18.190Z 24 480-499 68 68 68 130 130 130 F F F
76542024-01-10T17:58:18.190Z 25 500-519 70 68 70 133 130 133 F F F <---
76552024-01-10T17:58:18.190Z 26 520-539 70 66 70 133 128 133 F F F <---
76562024-01-10T17:58:18.190Z 27 540-559 64 64 64 126 126 126 F F F
76572024-01-10T17:58:18.190Z 28 560-579 64 64 64 126 126 126 F F F
76582024-01-10T17:58:18.190Z 29 580-599 70 64 70 133 126 133 F F F <---
76592024-01-10T17:58:18.190ZMax gen: 70, Max flush: 133
76602024-01-10T17:58:18.190ZError: Difference in extent metadata found!
76612024-01-10T17:58:18.194ZA Difference in extent metadata is expected here
76622024-01-10T17:58:18.194ZOn loop 35
76632024-01-10T17:58:18.194Z
76642024-01-10T17:58:18.194ZVerifying data now
76652024-01-10T17:58:23.364ZNo change after verify
76662024-01-10T17:58:23.364ZLoop: 35 Downstairs dump after verify (and repair):
76672024-01-10T17:58:23.386ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76682024-01-10T17:58:23.386Z 0 000-019 67 67 67 129 129 129 F F F
76692024-01-10T17:58:23.386Z 1 020-039 69 69 69 131 131 131 F F F
76702024-01-10T17:58:23.386Z 2 040-059 70 70 70 132 132 132 F F F
76712024-01-10T17:58:23.386Z 3 060-079 70 70 70 132 132 132 F F F
76722024-01-10T17:58:23.386Z 4 080-099 70 70 70 133 133 133 F F F
76732024-01-10T17:58:23.386Z 5 100-119 64 64 64 126 126 126 F F F
76742024-01-10T17:58:23.386Z 6 120-139 60 60 60 117 117 117 F F F
76752024-01-10T17:58:23.386Z 7 140-159 71 71 71 134 134 134 F F F
76762024-01-10T17:58:23.386Z 8 160-179 64 64 64 125 125 125 F F F
76772024-01-10T17:58:23.386Z 9 180-199 67 67 67 129 129 129 F F F
76782024-01-10T17:58:23.386Z 10 200-219 70 70 70 133 133 133 F F F
76792024-01-10T17:58:23.387Z 11 220-239 67 67 67 129 129 129 F F F
76802024-01-10T17:58:23.387Z 12 240-259 67 67 67 129 129 129 F F F
76812024-01-10T17:58:23.390Z 13 260-279 70 70 70 133 133 133 F F F
76822024-01-10T17:58:23.390Z 14 280-299 70 70 70 132 132 132 F F F
76832024-01-10T17:58:23.390Z 15 300-319 68 68 68 130 130 130 F F F
76842024-01-10T17:58:23.390Z 16 320-339 70 70 70 132 132 132 F F F
76852024-01-10T17:58:23.390Z 17 340-359 68 68 68 130 130 130 F F F
76862024-01-10T17:58:23.390Z 18 360-379 58 58 58 111 111 111 F F F
76872024-01-10T17:58:23.391Z 19 380-399 70 70 70 132 132 132 F F F
76882024-01-10T17:58:23.391Z 20 400-419 68 68 68 130 130 130 F F F
76892024-01-10T17:58:23.391Z 21 420-439 68 68 68 130 130 130 F F F
76902024-01-10T17:58:23.391Z 22 440-459 67 67 67 129 129 129 F F F
76912024-01-10T17:58:23.391Z 23 460-479 63 63 63 124 124 124 F F F
76922024-01-10T17:58:23.391Z 24 480-499 68 68 68 130 130 130 F F F
76932024-01-10T17:58:23.391Z 25 500-519 70 70 70 133 133 133 F F F
76942024-01-10T17:58:23.391Z 26 520-539 70 70 70 133 133 133 F F F
76952024-01-10T17:58:23.391Z 27 540-559 64 64 64 126 126 126 F F F
76962024-01-10T17:58:23.391Z 28 560-579 64 64 64 126 126 126 F F F
76972024-01-10T17:58:23.391Z 29 580-599 70 70 70 133 133 133 F F F
76982024-01-10T17:58:23.391ZMax gen: 71, Max flush: 134
76992024-01-10T17:58:23.391Z
77002024-01-10T17:58:23.391ZBegin loop 36
77012024-01-10T17:58:23.391ZDownstairs to restart: 0
77022024-01-10T17:58:23.420Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.421772307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443}
77032024-01-10T17:58:23.420Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.422280348Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443}
77042024-01-10T17:58:23.425Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.422313375Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443}
77052024-01-10T17:58:23.425ZCrucible runtime is spawned
77062024-01-10T17:58:23.425Z{"msg":"Crucible 3946a649-51c6-4719-b6ac-39a0a2fa420c has session id: cfe2063d-7de4-457f-9781-67222f5876ec","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.422510726Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77072024-01-10T17:58:23.425Z{"msg":"Upstairs opts: Upstairs UUID: 3946a649-51c6-4719-b6ac-39a0a2fa420c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.422597418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77082024-01-10T17:58:23.425Z{"msg":"Crucible stats registered with UUID: 3946a649-51c6-4719-b6ac-39a0a2fa420c","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.422630167Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77092024-01-10T17:58:23.425Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.422651786Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443}
77102024-01-10T17:58:23.425Z{"msg":"3946a649-51c6-4719-b6ac-39a0a2fa420c active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.422801778Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77112024-01-10T17:58:23.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.422860721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"0","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77122024-01-10T17:58:23.425Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.422886172Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"0","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77132024-01-10T17:58:23.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.422956717Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"1","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77142024-01-10T17:58:23.425Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.423002402Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"1","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77152024-01-10T17:58:23.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.423097975Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"2","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77162024-01-10T17:58:23.425Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.423147528Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"2","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77172024-01-10T17:58:23.426Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.423302162Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77182024-01-10T17:58:23.426Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.423433584Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77192024-01-10T17:58:23.426Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.423584975Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77202024-01-10T17:58:23.426Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.423632309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77212024-01-10T17:58:23.426Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.423782994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77222024-01-10T17:58:23.426Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.423888713Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77232024-01-10T17:58:23.426Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.424857701Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"1","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77242024-01-10T17:58:23.427Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.424922636Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"1","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77252024-01-10T17:58:23.427Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.424949422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"0","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77262024-01-10T17:58:23.427Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.424973323Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"0","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77272024-01-10T17:58:23.427Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.425014262Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"2","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77282024-01-10T17:58:23.427Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.425049884Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"2","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77292024-01-10T17:58:23.427Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.426055863Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"2","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77302024-01-10T17:58:23.427Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.426133629Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"0","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77312024-01-10T17:58:23.427Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.426293013Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"1","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77322024-01-10T17:58:23.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427010039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"2","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77332024-01-10T17:58:23.431Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427118628Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"2","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77342024-01-10T17:58:23.431Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427145973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77352024-01-10T17:58:23.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427217775Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"1","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77362024-01-10T17:58:23.431Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427260557Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"1","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77372024-01-10T17:58:23.431Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.42734967Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77382024-01-10T17:58:23.432Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427390773Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"0","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77392024-01-10T17:58:23.432Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427436262Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"0","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77402024-01-10T17:58:23.432Z{"msg":"[0]R flush_numbers[0..12]: [129, 131, 132, 132, 133, 126, 117, 134, 125, 129, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427461375Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77412024-01-10T17:58:23.432Z{"msg":"[0]R generation[0..12]: [67, 69, 70, 70, 70, 64, 60, 71, 64, 67, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427484204Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77422024-01-10T17:58:23.432Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427507564Z","hostname":"Wait for a query_work_queue command to finish before sending IO
77432024-01-10T17:58:23.432Zip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77442024-01-10T17:58:23.432Z{"msg":"[1]R flush_numbers[0..12]: [129, 131, 132, 132, 133, 126, 117, 134, 125, 129, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.42761521Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77452024-01-10T17:58:23.432Z{"msg":"[1]R generation[0..12]: [67, 69, 70, 70, 70, 64, 60, 71, 64, 67, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427692998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77462024-01-10T17:58:23.432Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.42771911Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77472024-01-10T17:58:23.432Z{"msg":"[2]R flush_numbers[0..12]: [129, 131, 132, 132, 133, 126, 117, 134, 125, 129, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427742429Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77482024-01-10T17:58:23.432Z{"msg":"[2]R generation[0..12]: [67, 69, 70, 70, 70, 64, 60, 71, 64, 67, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427783801Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77492024-01-10T17:58:23.432Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427808402Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77502024-01-10T17:58:23.432Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427829803Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77512024-01-10T17:58:23.432Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.42785253Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77522024-01-10T17:58:23.432Z{"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427873556Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77532024-01-10T17:58:23.432Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427912647Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"":"downstairs","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77542024-01-10T17:58:23.432Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427935624Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77552024-01-10T17:58:23.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.42795557Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"0","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77562024-01-10T17:58:23.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.427977037Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"1","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77572024-01-10T17:58:23.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.428001851Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"client":"2","":"downstairs","session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77582024-01-10T17:58:23.432Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.428046807Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77592024-01-10T17:58:23.432Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.428068489Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77602024-01-10T17:58:23.432Z{"msg":"3946a649-51c6-4719-b6ac-39a0a2fa420c is now active with session: cfe2063d-7de4-457f-9781-67222f5876ec","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.428089771Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77612024-01-10T17:58:23.433Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.428108652Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443,"session_id":"cfe2063d-7de4-457f-9781-67222f5876ec"}
77622024-01-10T17:58:23.433Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:23.428129996Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1443}
77632024-01-10T17:58:23.433ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
77642024-01-10T17:58:23.433ZLoading write count information from file "/tmp/test_repair_verify.data"
77652024-01-10T17:58:23.433ZRun Repair workload
77662024-01-10T17:58:23.433Z01/30 Write block 34 len 20480 data: 7 8 8 7 7
77672024-01-10T17:58:23.434Z02/30 Write block 387 len 8192 data: 4 3
77682024-01-10T17:58:23.434Z03/30 Read block 228 len 24576
77692024-01-10T17:58:23.437Z04/30 Read block 382 len 4096
77702024-01-10T17:58:23.437Z05/30 Read block 69 len 28672
77712024-01-10T17:58:23.437Z06/30 Read block 471 len 12288
77722024-01-10T17:58:23.440Z07/30 Write block 149 len 8192 data: 7 8
77732024-01-10T17:58:23.440Z08/30 Write block 304 len 8192 data: 7 6
77742024-01-10T17:58:23.440Z09/30 Write block 185 len 4096 data: 8
77752024-01-10T17:58:23.440Z10/30 Read block 597 len 8192
77762024-01-10T17:58:23.443Z11/30 Read block 375 len 12288
77772024-01-10T17:58:23.443Z12/30 Write block 99 len 40960 data: 7 6 5 5 4 4 5 5 4 5
77782024-01-10T17:58:23.446Z13/30 Read block 386 len 16384
77792024-01-10T17:58:23.450Z14/30 Read block 326 len 32768
77802024-01-10T17:58:23.453Z15/30 Write block 457 len 32768 data: 4 4 4 5 5 5 6 5
77812024-01-10T17:58:23.453Z16/30 Write block 83 len 32768 data: 7 5 4 4 4 4 4 4
77822024-01-10T17:58:23.453Z17/30 Read block 215 len 36864
77832024-01-10T17:58:23.457Z18/30 Write block 284 len 24576 data: 8 8 6 7 6 8
77842024-01-10T17:58:23.460Z19/30 Write block 169 len 8192 data: 8 6
77852024-01-10T17:58:23.460Z20/30 Read block 241 len 20480
77862024-01-10T17:58:23.463Z21/30 Read block 70 len 36864
77872024-01-10T17:58:23.466Z22/30 Read block 349 len 28672
77882024-01-10T17:58:23.469Z23/30 Read block 223 len 28672
77892024-01-10T17:58:23.469Z24/30 Read block 343 len 36864
77902024-01-10T17:58:23.472Z25/30 Write block 139 len 24576 data: 6 7 8 6 6 6
77912024-01-10T17:58:23.475Z26/30 Flush
77922024-01-10T17:58:23.479Z27/30 Write block 147 len 8192 data: 7 7
77932024-01-10T17:58:23.482Z28/30 Read block 90 len 36864
77942024-01-10T17:58:23.484Z29/30 Write block 26 len 36864 data: 7 7 7 8 11 9 9 8 8
77952024-01-10T17:58:23.484Z30/30 Read block 412 len 40960
77962024-01-10T17:58:23.488Z----------------------------------------------------------------
77972024-01-10T17:58:23.488Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
77982024-01-10T17:58:23.488ZStates: Active Active Active
77992024-01-10T17:58:23.488ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78002024-01-10T17:58:23.488Z 1 Acked 1000 Write 5 Done Done Done false
78012024-01-10T17:58:23.488Z 2 Acked 1001 Write 2 Done Done Done false
78022024-01-10T17:58:23.488Z 3 Acked 1002 Read 6 Done Done Done false
78032024-01-10T17:58:23.488Z 4 Acked 1003 Read 1 Sent Done Done false
78042024-01-10T17:58:23.489Z 5 Acked 1004 Read 7 Sent Done Done false
78052024-01-10T17:58:23.489Z 6 Acked 1005 Read 3 Sent Done Done false
78062024-01-10T17:58:23.489Z 7 Acked 1006 Write 2 Sent Done Done false
78072024-01-10T17:58:23.489Z 8 Acked 1007 Write 2 Sent Done Done false
78082024-01-10T17:58:23.489Z 9 Acked 1008 Write 1 Sent Done Done false
78092024-01-10T17:58:23.489Z 10 Acked 1009 Read 2 Sent Done Done false
78102024-01-10T17:58:23.489Z 11 Acked 1010 Read 3 Sent Done Done false
78112024-01-10T17:58:23.489Z 12 Acked 1011 Write 10 Sent Done Done false
78122024-01-10T17:58:23.489Z 13 Acked 1012 Read 4 Sent Done Done false
78132024-01-10T17:58:23.489Z 14 Acked 1013 Read 8 Sent Done Done false
78142024-01-10T17:58:23.489Z 15 Acked 1014 Write 8 Sent Done Done false
78152024-01-10T17:58:23.489Z 16 Acked 1015 Write 8 Sent Done Done false
78162024-01-10T17:58:23.490Z 17 Acked 1016 Read 9 Sent Done Done false
78172024-01-10T17:58:23.490Z 18 Acked 1017 Write 6 Sent Done Done false
78182024-01-10T17:58:23.490Z 19 Acked 1018 Write 2 Sent Done Done false
78192024-01-10T17:58:23.490Z 20 Acked 1019 Read 5 Sent Done Done false
78202024-01-10T17:58:23.490Z 21 Acked 1020 Read 9 Sent Done Done false
78212024-01-10T17:58:23.490Z 22 Acked 1021 Read 7 Sent Done Done false
78222024-01-10T17:58:23.490Z 23 Acked 1022 Read 7 Sent Done Done false
78232024-01-10T17:58:23.491Z 24 Acked 1023 Read 9 Sent Done Done false
78242024-01-10T17:58:23.491Z 25 Acked 1024 Write 6 Sent Done Done false
78252024-01-10T17:58:23.491Z 26 Acked 1025 Flush 0 Sent Done Done false
78262024-01-10T17:58:23.491Z 27 Acked 1026 Write 2 Sent Done Done false
78272024-01-10T17:58:23.491Z 28 Acked 1027 Read 9 Sent Done Done false
78282024-01-10T17:58:23.491Z 29 Acked 1028 Write 9 Sent Done Done false
78292024-01-10T17:58:23.491Z 30 Acked 1029 Read 10 Sent Done Sent false
78302024-01-10T17:58:23.491Z STATES DS:0 DS:1 DS:2 TOTAL
78312024-01-10T17:58:23.494Z New 0 0 0 0
78322024-01-10T17:58:23.494Z Sent 27 0 1 28
78332024-01-10T17:58:23.494Z Done 3 30 29 62
78342024-01-10T17:58:23.494Z Skipped 0 0 0 0
78352024-01-10T17:58:23.494Z Error 0 0 0 0
78362024-01-10T17:58:23.494ZLast Flush: 0 1025 1025
78372024-01-10T17:58:23.494ZDownstairs last five completed:
78382024-01-10T17:58:23.494ZUpstairs last five completed: 30 29 28 27 26
78392024-01-10T17:58:23.495ZWrote out file "/tmp/test_repair_verify.data"
78402024-01-10T17:58:23.499Z
78412024-01-10T17:58:23.504ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78422024-01-10T17:58:23.524ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78432024-01-10T17:58:23.524Z 0 000-019 67 67 67 129 129 129 F F F
78442024-01-10T17:58:23.524Z 1 020-039 69 72 72 131 135 135 T T T <---
78452024-01-10T17:58:23.524Z 2 040-059 70 70 70 132 132 132 F F F
78462024-01-10T17:58:23.524Z 3 060-079 70 70 70 132 132 132 F F F
78472024-01-10T17:58:23.527Z 4 080-099 70 72 72 133 135 135 F F F <---
78482024-01-10T17:58:23.527Z 5 100-119 64 72 72 126 135 135 F F F <---
78492024-01-10T17:58:23.527Z 6 120-139 60 72 72 117 135 135 F F F <---
78502024-01-10T17:58:23.528Z 7 140-159 71 72 72 134 135 135 F T T <---
78512024-01-10T17:58:23.528Z 8 160-179 64 72 72 125 135 135 F F F <---
78522024-01-10T17:58:23.528Z 9 180-199 67 72 72 129 135 135 F F F <---
78532024-01-10T17:58:23.528Z 10 200-219 70 70 70 133 133 133 F F F
78542024-01-10T17:58:23.528Z 11 220-239 67 67 67 129 129 129 F F F
78552024-01-10T17:58:23.528Z 12 240-259 67 67 67 129 129 129 F F F
78562024-01-10T17:58:23.528Z 13 260-279 70 70 70 133 133 133 F F F
78572024-01-10T17:58:23.528Z 14 280-299 70 72 72 132 135 135 F F F <---
78582024-01-10T17:58:23.528Z 15 300-319 68 72 72 130 135 135 F F F <---
78592024-01-10T17:58:23.528Z 16 320-339 70 70 70 132 132 132 F F F
78602024-01-10T17:58:23.528Z 17 340-359 68 68 68 130 130 130 F F F
78612024-01-10T17:58:23.528Z 18 360-379 58 58 58 111 111 111 F F F
78622024-01-10T17:58:23.528Z 19 380-399 70 72 72 132 135 135 T F F <---
78632024-01-10T17:58:23.528Z 20 400-419 68 68 68 130 130 130 F F F
78642024-01-10T17:58:23.529Z 21 420-439 68 68 68 130 130 130 F F F
78652024-01-10T17:58:23.529Z 22 440-459 67 72 72 129 135 135 F F F <---
78662024-01-10T17:58:23.529Z 23 460-479 63 72 72 124 135 135 F F F <---
78672024-01-10T17:58:23.529Z 24 480-499 68 68 68 130 130 130 F F F
78682024-01-10T17:58:23.529Z 25 500-519 70 70 70 133 133 133 F F F
78692024-01-10T17:58:23.529Z 26 520-539 70 70 70 133 133 133 F F F
78702024-01-10T17:58:23.529Z 27 540-559 64 64 64 126 126 126 F F F
78712024-01-10T17:58:23.529Z 28 560-579 64 64 64 126 126 126 F F F
78722024-01-10T17:58:23.529Z 29 580-599 70 70 70 133 133 133 F F F
78732024-01-10T17:58:23.529ZMax gen: 72, Max flush: 135
78742024-01-10T17:58:23.529ZError: Difference in extent metadata found!
78752024-01-10T17:58:23.533ZA Difference in extent metadata is expected here
78762024-01-10T17:58:23.533ZOn loop 36
78772024-01-10T17:58:23.533Z
78782024-01-10T17:58:23.533ZVerifying data now
78792024-01-10T17:58:28.710ZNo change after verify
78802024-01-10T17:58:28.710ZLoop: 36 Downstairs dump after verify (and repair):
78812024-01-10T17:58:28.732ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78822024-01-10T17:58:28.733Z 0 000-019 67 67 67 129 129 129 F F F
78832024-01-10T17:58:28.733Z 1 020-039 73 73 73 136 136 136 F F F
78842024-01-10T17:58:28.733Z 2 040-059 70 70 70 132 132 132 F F F
78852024-01-10T17:58:28.733Z 3 060-079 70 70 70 132 132 132 F F F
78862024-01-10T17:58:28.733Z 4 080-099 72 72 72 135 135 135 F F F
78872024-01-10T17:58:28.734Z 5 100-119 72 72 72 135 135 135 F F F
78882024-01-10T17:58:28.742Z 6 120-139 72 72 72 135 135 135 F F F
78892024-01-10T17:58:28.742Z 7 140-159 73 73 73 136 136 136 F F F
78902024-01-10T17:58:28.742Z 8 160-179 72 72 72 135 135 135 F F F
78912024-01-10T17:58:28.742Z 9 180-199 72 72 72 135 135 135 F F F
78922024-01-10T17:58:28.742Z 10 200-219 70 70 70 133 133 133 F F F
78932024-01-10T17:58:28.742Z 11 220-239 67 67 67 129 129 129 F F F
78942024-01-10T17:58:28.743Z 12 240-259 67 67 67 129 129 129 F F F
78952024-01-10T17:58:28.743Z 13 260-279 70 70 70 133 133 133 F F F
78962024-01-10T17:58:28.744Z 14 280-299 72 72 72 135 135 135 F F F
78972024-01-10T17:58:28.744Z 15 300-319 72 72 72 135 135 135 F F F
78982024-01-10T17:58:28.744Z 16 320-339 70 70 70 132 132 132 F F F
78992024-01-10T17:58:28.744Z 17 340-359 68 68 68 130 130 130 F F F
79002024-01-10T17:58:28.744Z 18 360-379 58 58 58 111 111 111 F F F
79012024-01-10T17:58:28.744Z 19 380-399 72 72 72 135 135 135 F F F
79022024-01-10T17:58:28.744Z 20 400-419 68 68 68 130 130 130 F F F
79032024-01-10T17:58:28.744Z 21 420-439 68 68 68 130 130 130 F F F
79042024-01-10T17:58:28.744Z 22 440-459 72 72 72 135 135 135 F F F
79052024-01-10T17:58:28.744Z 23 460-479 72 72 72 135 135 135 F F F
79062024-01-10T17:58:28.744Z 24 480-499 68 68 68 130 130 130 F F F
79072024-01-10T17:58:28.744Z 25 500-519 70 70 70 133 133 133 F F F
79082024-01-10T17:58:28.744Z 26 520-539 70 70 70 133 133 133 F F F
79092024-01-10T17:58:28.745Z 27 540-559 64 64 64 126 126 126 F F F
79102024-01-10T17:58:28.745Z 28 560-579 64 64 64 126 126 126 F F F
79112024-01-10T17:58:28.745Z 29 580-599 70 70 70 133 133 133 F F F
79122024-01-10T17:58:28.745ZMax gen: 73, Max flush: 136
79132024-01-10T17:58:28.745Z
79142024-01-10T17:58:28.745ZBegin loop 37
79152024-01-10T17:58:28.745ZDownstairs to restart: 2
79162024-01-10T17:58:28.768Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.765588056Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451}
79172024-01-10T17:58:28.768Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766118684Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451}
79182024-01-10T17:58:28.775Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766177763Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451}
79192024-01-10T17:58:28.775Z{"msg":"Crucible c1afb5d2-88fa-4061-a3e0-4a03612b720c has session id: bd7e25d3-2210-4afa-b512-82a83ad608c6","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.76625861Z","hostnameCrucible runtime is spawned
79202024-01-10T17:58:28.776Z":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79212024-01-10T17:58:28.776Z{"msg":"Upstairs opts: Upstairs UUID: c1afb5d2-88fa-4061-a3e0-4a03612b720c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766414618Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79222024-01-10T17:58:28.776Z{"msg":"Crucible stats registered with UUID: c1afb5d2-88fa-4061-a3e0-4a03612b720c","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.76647345Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79232024-01-10T17:58:28.777Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766498072Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451}
79242024-01-10T17:58:28.777Z{"msg":"c1afb5d2-88fa-4061-a3e0-4a03612b720c active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766658899Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79252024-01-10T17:58:28.777Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766708997Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"0","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79262024-01-10T17:58:28.777Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766734212Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"0","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79272024-01-10T17:58:28.777Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766799881Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"1","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79282024-01-10T17:58:28.778Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766843285Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"1","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79292024-01-10T17:58:28.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766899098Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"2","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79302024-01-10T17:58:28.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766942565Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"2","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79312024-01-10T17:58:28.780Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.766995095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79322024-01-10T17:58:28.781Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.767039172Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79332024-01-10T17:58:28.781Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.767104364Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79342024-01-10T17:58:28.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.767373849Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79352024-01-10T17:58:28.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.767691019Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79362024-01-10T17:58:28.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.76778051Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79372024-01-10T17:58:28.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.768776469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"0","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79382024-01-10T17:58:28.782Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.768852443Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"0","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79392024-01-10T17:58:28.782Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.768896665Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"1","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79402024-01-10T17:58:28.782Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.768938503Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"1","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79412024-01-10T17:58:28.782Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.769673601Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"2","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79422024-01-10T17:58:28.782Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.769829245Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"2","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79432024-01-10T17:58:28.783Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.769979276Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"1","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79442024-01-10T17:58:28.783Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.770071206Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"0","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79452024-01-10T17:58:28.783Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.77060233Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"2","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79462024-01-10T17:58:28.783Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.770866136Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"1","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79472024-01-10T17:58:28.783Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.770944606Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"1","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79482024-01-10T17:58:28.785Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.770990905Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79492024-01-10T17:58:28.785Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.771037233Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"0","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79502024-01-10T17:58:28.786Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.771111178Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"0","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79512024-01-10T17:58:28.786Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.77115582Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79522024-01-10T17:58:28.786Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.771588862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"2","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79532024-01-10T17:58:28.786Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.771648304Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"2","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79542024-01-10T17:58:28.786Z{"msg":"[0]R flush_numbers[0..12]: [129, 136, 132, 132, 135, 135, 135, 136, 135, 135, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.771674954Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79552024-01-10T17:58:28.786Z{"msg":"[0]R generation[0..12]: [67, 73, 70, 70, 72, 72, 72, 73, 72, 72, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.771734106Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79562024-01-10T17:58:28.787Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.771768337Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79572024-01-10T17:58:28.787Z{"msg":"[1]R flush_numbers[0..12]: [129, 136, 132, 132, 135, 135, 135, 136, 135, 135, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.771835722Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79582024-01-10T17:58:28.787Z{"msg":"[1]R generation[0..12]: [67, 73, 70, 70, 72, 72, 72, 73, 72, 72, 70, 67]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
79592024-01-10T17:58:28.787Ztime":"2024-01-10T17:58:28.771861738Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79602024-01-10T17:58:28.787Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772010173Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79612024-01-10T17:58:28.788Z{"msg":"[2]R flush_numbers[0..12]: [129, 136, 132, 132, 135, 135, 135, 136, 135, 135, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772118967Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79622024-01-10T17:58:28.788Z{"msg":"[2]R generation[0..12]: [67, 73, 70, 70, 72, 72, 72, 73, 72, 72, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772219817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79632024-01-10T17:58:28.788Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772285324Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79642024-01-10T17:58:28.788Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772311092Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79652024-01-10T17:58:28.788Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.77237353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79662024-01-10T17:58:28.789Z{"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772420222Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79672024-01-10T17:58:28.789Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772483969Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"":"downstairs","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79682024-01-10T17:58:28.790Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772509603Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79692024-01-10T17:58:28.790Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772572472Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"0","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79702024-01-10T17:58:28.790Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772598405Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"1","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79712024-01-10T17:58:28.790Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
79722024-01-10T17:58:28.791Z,"time":"2024-01-10T17:58:28.772672024Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"client":"2","":"downstairs","session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79732024-01-10T17:58:28.791Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772698501Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79742024-01-10T17:58:28.791Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.77273002Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79752024-01-10T17:58:28.791Z{"msg":"c1afb5d2-88fa-4061-a3e0-4a03612b720c is now active with session: bd7e25d3-2210-4afa-b512-82a83ad608c6","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772759753Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79762024-01-10T17:58:28.791Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772779575Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451,"session_id":"bd7e25d3-2210-4afa-b512-82a83ad608c6"}
79772024-01-10T17:58:28.791Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:28.772805046Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1451}
79782024-01-10T17:58:28.792ZLoading write count information from file "/tmp/test_repair_verify.data"
79792024-01-10T17:58:28.792ZRun Repair workload
79802024-01-10T17:58:28.793Z01/30 Write block 95 len 12288 data: 7 7 7
79812024-01-10T17:58:28.804Z02/30 Read block 525 len 32768
79822024-01-10T17:58:28.804Z03/30 Flush
79832024-01-10T17:58:28.804Z04/30 Write block 91 len 28672 data: 7 6 6 5 8 8 8
79842024-01-10T17:58:28.804Z05/30 Read block 588 len 8192
79852024-01-10T17:58:28.804Z06/30 Write block 76 len 4096 data: 7
79862024-01-10T17:58:28.804Z07/30 Read block 263 len 28672
79872024-01-10T17:58:28.804Z08/30 Read block 271 len 4096
79882024-01-10T17:58:28.805Z09/30 Read block 494 len 32768
79892024-01-10T17:58:28.805Z10/30 Read block 341 len 12288
79902024-01-10T17:58:28.805Z11/30 Write block 45 len 4096 data: 5
79912024-01-10T17:58:28.805Z12/30 Write block 54 len 20480 data: 7 7 6 5 7
79922024-01-10T17:58:28.805Z13/30 Flush
79932024-01-10T17:58:28.805Z14/30 Write block 315 len 4096 data: 6
79942024-01-10T17:58:28.805Z15/30 Write block 391 len 40960 data: 5 5 4 5 5 5 5 6 8 5
79952024-01-10T17:58:28.806Z16/30 Write block 428 len 28672 data: 5 5 3 3 3 4 4
79962024-01-10T17:58:28.806Z17/30 Write block 330 len 8192 data: 6 5
79972024-01-10T17:58:28.810Z18/30 Read block 208 len 40960
79982024-01-10T17:58:28.810Z19/30 Read block 333 len 36864
79992024-01-10T17:58:28.810Z20/30 Read block 485 len 24576
80002024-01-10T17:58:28.814Z21/30 Write block 270 len 28672 data: 8 8 6 7 10 11 9
80012024-01-10T17:58:28.814Z22/30 Read block 202 len 40960
80022024-01-10T17:58:28.819Z23/30 Write block 513 len 40960 data: 11 12 10 9 9 8 7 7 7 5
80032024-01-10T17:58:28.819Z24/30 Read block 38 len 40960
80042024-01-10T17:58:28.824Z25/30 Write block 530 len 28672 data: 10 12 10 10 9 8 8
80052024-01-10T17:58:28.824Z26/30 Write block 352 len 32768 data: 5 4 4 3 4 5 7 6
80062024-01-10T17:58:28.832Z27/30 Read block 222 len 8192
80072024-01-10T17:58:28.832Z28/30 Write block 156 len 12288 data: 5 4 2
80082024-01-10T17:58:28.832Z29/30 Write block 446 len 16384 data: 12 10 8 7
80092024-01-10T17:58:28.832Z30/30 Read block 88 len 32768
80102024-01-10T17:58:28.838Z----------------------------------------------------------------
80112024-01-10T17:58:28.838Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
80122024-01-10T17:58:28.838ZStates: Active Active Active
80132024-01-10T17:58:28.838ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80142024-01-10T17:58:28.838Z 1 Acked 1000 Write 3 Done Done Done false
80152024-01-10T17:58:28.838Z 2 Acked 1001 Read 8 Done Done Done false
80162024-01-10T17:58:28.838Z 3 Acked 1002 Flush 0 Done Done Sent false
80172024-01-10T17:58:28.838Z 4 Acked 1003 Write 7 Done Done Sent false
80182024-01-10T17:58:28.839Z 5 Acked 1004 Read 2 Done Done Sent false
80192024-01-10T17:58:28.839Z 6 Acked 1005 Write 1 Done Done Sent false
80202024-01-10T17:58:28.839Z 7 Acked 1006 Read 7 Done Done Sent false
80212024-01-10T17:58:28.839Z 8 Acked 1007 Read 1 Done Done Sent false
80222024-01-10T17:58:28.839Z 9 Acked 1008 Read 8 Done Done Sent false
80232024-01-10T17:58:28.839Z 10 Acked 1009 Read 3 Done Done Sent false
80242024-01-10T17:58:28.839Z 11 Acked 1010 Write 1 Done Done Sent false
80252024-01-10T17:58:28.839Z 12 Acked 1011 Write 5 Done Done Sent false
80262024-01-10T17:58:28.840Z 13 Acked 1012 Flush 0 Done Done Sent false
80272024-01-10T17:58:28.840Z 14 Acked 1013 Write 1 Done Done Sent false
80282024-01-10T17:58:28.840Z 15 Acked 1014 Write 10 Done Done Sent false
80292024-01-10T17:58:28.840Z 16 Acked 1015 Write 7 Done Done Sent false
80302024-01-10T17:58:28.840Z 17 Acked 1016 Write 2 Done Done Sent false
80312024-01-10T17:58:28.840Z 18 Acked 1017 Read 10 Done Done Sent false
80322024-01-10T17:58:28.840Z 19 Acked 1018 Read 9 Done Done Sent false
80332024-01-10T17:58:28.840Z 20 Acked 1019 Read 6 Done Done Sent false
80342024-01-10T17:58:28.840Z 21 Acked 1020 Write 7 Done Done Sent false
80352024-01-10T17:58:28.840Z 22 Acked 1021 Read 10 Done Done Sent false
80362024-01-10T17:58:28.840Z 23 Acked 1022 Write 10 Done Done Sent false
80372024-01-10T17:58:28.840Z 24 Acked 1023 Read 10 Done Done Sent false
80382024-01-10T17:58:28.841Z 25 Acked 1024 Write 7 Done Done Sent false
80392024-01-10T17:58:28.841Z 26 Acked 1025 Write 8 Done Done Sent false
80402024-01-10T17:58:28.841Z 27 Acked 1026 Read 2 Done Done Sent false
80412024-01-10T17:58:28.841Z 28 Acked 1027 Write 3 Done Done Sent false
80422024-01-10T17:58:28.841Z 29 Acked 1028 Write 4 Done Done Sent false
80432024-01-10T17:58:28.841Z 30 Acked 1029 Read 8 Done Sent Sent false
80442024-01-10T17:58:28.841Z STATES DS:0 DS:1 DS:2 TOTAL
80452024-01-10T17:58:28.841Z New 0 0 0 0
80462024-01-10T17:58:28.841Z Sent 0 1 28 29
80472024-01-10T17:58:28.841Z Done 30 29 2 61
80482024-01-10T17:58:28.841Z Skipped 0 0 0 0
80492024-01-10T17:58:28.841Z Error 0 0 0 0
80502024-01-10T17:58:28.842ZLast Flush: 1012 1012 0
80512024-01-10T17:58:28.842ZDownstairs last five completed:
80522024-01-10T17:58:28.842ZUpstairs last five completed: 30 29 28 27 26
80532024-01-10T17:58:28.842ZWrote out file "/tmp/test_repair_verify.data"
80542024-01-10T17:58:28.849Z
80552024-01-10T17:58:28.849ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
80562024-01-10T17:58:28.871ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80572024-01-10T17:58:28.872Z 0 000-019 67 67 67 129 129 129 F F F
80582024-01-10T17:58:28.872Z 1 020-039 73 73 73 136 136 136 F F F
80592024-01-10T17:58:28.879Z 2 040-059 74 74 70 138 138 132 F F F <---
80602024-01-10T17:58:28.879Z 3 060-079 74 74 70 138 138 132 F F F <---
80612024-01-10T17:58:28.879Z 4 080-099 74 74 72 138 138 135 F F T <---
80622024-01-10T17:58:28.879Z 5 100-119 72 72 72 135 135 135 F F F
80632024-01-10T17:58:28.879Z 6 120-139 72 72 72 135 135 135 F F F
80642024-01-10T17:58:28.879Z 7 140-159 73 73 73 136 136 136 T T F <---
80652024-01-10T17:58:28.879Z 8 160-179 72 72 72 135 135 135 F F F
80662024-01-10T17:58:28.879Z 9 180-199 72 72 72 135 135 135 F F F
80672024-01-10T17:58:28.880Z 10 200-219 70 70 70 133 133 133 F F F
80682024-01-10T17:58:28.880Z 11 220-239 67 67 67 129 129 129 F F F
80692024-01-10T17:58:28.880Z 12 240-259 67 67 67 129 129 129 F F F
80702024-01-10T17:58:28.880Z 13 260-279 70 70 70 133 133 133 T T F <---
80712024-01-10T17:58:28.880Z 14 280-299 72 72 72 135 135 135 F F F
80722024-01-10T17:58:28.880Z 15 300-319 72 72 72 135 135 135 T T F <---
80732024-01-10T17:58:28.880Z 16 320-339 70 70 70 132 132 132 T T F <---
80742024-01-10T17:58:28.880Z 17 340-359 68 68 68 130 130 130 T T F <---
80752024-01-10T17:58:28.880Z 18 360-379 58 58 58 111 111 111 F F F
80762024-01-10T17:58:28.881Z 19 380-399 72 72 72 135 135 135 T T F <---
80772024-01-10T17:58:28.881Z 20 400-419 68 68 68 130 130 130 T T F <---
80782024-01-10T17:58:28.882Z 21 420-439 68 68 68 130 130 130 T T F <---
80792024-01-10T17:58:28.882Z 22 440-459 72 72 72 135 135 135 T T F <---
80802024-01-10T17:58:28.882Z 23 460-479 72 72 72 135 135 135 F F F
80812024-01-10T17:58:28.882Z 24 480-499 68 68 68 130 130 130 F F F
80822024-01-10T17:58:28.882Z 25 500-519 70 70 70 133 133 133 T T F <---
80832024-01-10T17:58:28.882Z 26 520-539 70 70 70 133 133 133 T T F <---
80842024-01-10T17:58:28.882Z 27 540-559 64 64 64 126 126 126 F F F
80852024-01-10T17:58:28.883Z 28 560-579 64 64 64 126 126 126 F F F
80862024-01-10T17:58:28.883Z 29 580-599 70 70 70 133 133 133 F F F
80872024-01-10T17:58:28.883ZMax gen: 74, Max flush: 138
80882024-01-10T17:58:28.883ZError: Difference in extent metadata found!
80892024-01-10T17:58:28.887ZA Difference in extent metadata is expected here
80902024-01-10T17:58:28.887ZOn loop 37
80912024-01-10T17:58:28.887Z
80922024-01-10T17:58:28.887ZVerifying data now
80932024-01-10T17:58:34.274ZNo change after verify
80942024-01-10T17:58:34.274ZLoop: 37 Downstairs dump after verify (and repair):
80952024-01-10T17:58:34.302ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80962024-01-10T17:58:34.302Z 0 000-019 67 67 67 129 129 129 F F F
80972024-01-10T17:58:34.302Z 1 020-039 73 73 73 136 136 136 F F F
80982024-01-10T17:58:34.302Z 2 040-059 74 74 74 138 138 138 F F F
80992024-01-10T17:58:34.302Z 3 060-079 74 74 74 138 138 138 F F F
81002024-01-10T17:58:34.302Z 4 080-099 74 74 74 138 138 138 F F F
81012024-01-10T17:58:34.302Z 5 100-119 72 72 72 135 135 135 F F F
81022024-01-10T17:58:34.302Z 6 120-139 72 72 72 135 135 135 F F F
81032024-01-10T17:58:34.302Z 7 140-159 75 75 75 139 139 139 F F F
81042024-01-10T17:58:34.302Z 8 160-179 72 72 72 135 135 135 F F F
81052024-01-10T17:58:34.302Z 9 180-199 72 72 72 135 135 135 F F F
81062024-01-10T17:58:34.303Z 10 200-219 70 70 70 133 133 133 F F F
81072024-01-10T17:58:34.314Z 11 220-239 67 67 67 129 129 129 F F F
81082024-01-10T17:58:34.314Z 12 240-259 67 67 67 129 129 129 F F F
81092024-01-10T17:58:34.315Z 13 260-279 75 75 75 139 139 139 F F F
81102024-01-10T17:58:34.315Z 14 280-299 72 72 72 135 135 135 F F F
81112024-01-10T17:58:34.315Z 15 300-319 75 75 75 139 139 139 F F F
81122024-01-10T17:58:34.316Z 16 320-339 75 75 75 139 139 139 F F F
81132024-01-10T17:58:34.316Z 17 340-359 75 75 75 139 139 139 F F F
81142024-01-10T17:58:34.316Z 18 360-379 58 58 58 111 111 111 F F F
81152024-01-10T17:58:34.318Z 19 380-399 75 75 75 139 139 139 F F F
81162024-01-10T17:58:34.318Z 20 400-419 75 75 75 139 139 139 F F F
81172024-01-10T17:58:34.318Z 21 420-439 75 75 75 139 139 139 F F F
81182024-01-10T17:58:34.318Z 22 440-459 75 75 75 139 139 139 F F F
81192024-01-10T17:58:34.318Z 23 460-479 72 72 72 135 135 135 F F F
81202024-01-10T17:58:34.319Z 24 480-499 68 68 68 130 130 130 F F F
81212024-01-10T17:58:34.319Z 25 500-519 75 75 75 139 139 139 F F F
81222024-01-10T17:58:34.319Z 26 520-539 75 75 75 139 139 139 F F F
81232024-01-10T17:58:34.319Z 27 540-559 64 64 64 126 126 126 F F F
81242024-01-10T17:58:34.319Z 28 560-579 64 64 64 126 126 126 F F F
81252024-01-10T17:58:34.319Z 29 580-599 70 70 70 133 133 133 F F F
81262024-01-10T17:58:34.320ZMax gen: 75, Max flush: 139
81272024-01-10T17:58:34.320Z
81282024-01-10T17:58:34.320ZBegin loop 38
81292024-01-10T17:58:34.320ZDownstairs to restart: 0
81302024-01-10T17:58:34.758Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.335133699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459}
81312024-01-10T17:58:34.759Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.335622344Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459}
81322024-01-10T17:58:34.759Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.33566312Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459}
81332024-01-10T17:58:34.793Z{"msg":"Crucible 5c9934bb-1f2c-4719-ab2d-3e44ef32a256 has session id: f01f674e-5511-4713-b5f9-b6d0f4b9c10e","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.335734244Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81342024-01-10T17:58:34.793Z{"msg":"Upstairs opts: Upstairs UUID: 5c9934bb-1f2c-4719-ab2d-3e44ef32a256, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"Crucible runtime is spawned
81352024-01-10T17:58:34.793Zname":"crucible","level":30,"time":"2024-01-10T17:58:34.335883902Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81362024-01-10T17:58:34.794Z{"msg":"Crucible stats registered with UUID: 5c9934bb-1f2c-4719-ab2d-3e44ef32a256","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.335995801Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81372024-01-10T17:58:34.794Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.3360397Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459}
81382024-01-10T17:58:34.794Z{"msg":"5c9934bb-1f2c-4719-ab2d-3e44ef32a256 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.336235949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81392024-01-10T17:58:34.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.33634193Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"0","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81402024-01-10T17:58:34.794Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.33637064Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"0","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81412024-01-10T17:58:34.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.336404065Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"1","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81422024-01-10T17:58:34.794Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.336477377Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"1","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81432024-01-10T17:58:34.795Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.336533031Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"2","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81442024-01-10T17:58:34.795Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.336578276Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"2","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81452024-01-10T17:58:34.795Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.336756768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81462024-01-10T17:58:34.795Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.33681042Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81472024-01-10T17:58:34.795Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.33685547Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81482024-01-10T17:58:34.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.336928846Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81492024-01-10T17:58:34.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.337029513Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81502024-01-10T17:58:34.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.337073425Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81512024-01-10T17:58:34.796Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.338053657Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"2","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81522024-01-10T17:58:34.797Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.338140172Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"2","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81532024-01-10T17:58:34.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.338269788Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"1","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81542024-01-10T17:58:34.797Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.338341297Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"1","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81552024-01-10T17:58:34.797Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.339155363Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"2","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81562024-01-10T17:58:34.797Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.339218899Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"1","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81572024-01-10T17:58:34.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.33966323Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"0","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81582024-01-10T17:58:34.798Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.339727793Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"0","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81592024-01-10T17:58:34.798Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.340069348Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"2","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81602024-01-10T17:58:34.798Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.340171482Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"2","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81612024-01-10T17:58:34.798Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.340207923Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81622024-01-10T17:58:34.798Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.340340121Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"1","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81632024-01-10T17:58:34.798Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.340391366Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"1","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81642024-01-10T17:58:34.799Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.340453783Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81652024-01-10T17:58:34.799Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.340764673Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"0","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81662024-01-10T17:58:34.799Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341435654Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"0","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81672024-01-10T17:58:34.799Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341518934Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"0","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81682024-01-10T17:58:34.800Z{"msg":"[0]R flush_numbers[0..12]: [129, 136, 138, 138, 138, 135, 135, 139, 135, 135, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341595269Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81692024-01-10T17:58:34.800Z{"msg":"[0]R generation[0..12]: [67, 73, 74, 74, 74, 72, 72, 75, 72, 72, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341646287Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81702024-01-10T17:58:34.800Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341672542Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81712024-01-10T17:58:34.800Z{"msg":"[1]R flush_numbers[0..12]: [129, 136, 138, 138, 138, 135, 135, 139, 135, 135, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.34171446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
81722024-01-10T17:58:34.800Zsession_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81732024-01-10T17:58:34.800Z{"msg":"[1]R generation[0..12]: [67, 73, 74, 74, 74, 72, 72, 75, 72, 72, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341742394Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81742024-01-10T17:58:34.800Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341784305Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81752024-01-10T17:58:34.800Z{"msg":"[2]R flush_numbers[0..12]: [129, 136, 138, 138, 138, 135, 135, 139, 135, 135, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341808942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81762024-01-10T17:58:34.801Z{"msg":"[2]R generation[0..12]: [67, 73, 74, 74, 74, 72, 72, 75, 72, 72, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341849582Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81772024-01-10T17:58:34.801Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341874289Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81782024-01-10T17:58:34.802Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.34191473Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81792024-01-10T17:58:34.802Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.341938269Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81802024-01-10T17:58:34.802Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.34196088Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81812024-01-10T17:58:34.802Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.342012589Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"":"downstairs","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81822024-01-10T17:58:34.802Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.342035634Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81832024-01-10T17:58:34.802Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.342076624Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"0","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81842024-01-10T17:58:34.802Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.342099235Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"1","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81852024-01-10T17:58:34.802Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.342120633Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"client":"2","":"downstairs","session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81862024-01-10T17:58:34.802Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.342159938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81872024-01-10T17:58:34.803Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.342181245Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81882024-01-10T17:58:34.804Z{"msg":"5c9934bb-1f2c-4719-ab2d-3e44ef32a256 is now active with session: f01f674e-5511-4713-b5f9-b6d0f4b9c10e","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.34220281Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81892024-01-10T17:58:34.804Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.342239744Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459,"session_id":"f01f674e-5511-4713-b5f9-b6d0f4b9c10e"}
81902024-01-10T17:58:34.804Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:34.342261151Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1459}
81912024-01-10T17:58:34.804ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
81922024-01-10T17:58:34.804ZLoading write count information from file "/tmp/test_repair_verify.data"
81932024-01-10T17:58:34.804ZRun Repair workload
81942024-01-10T17:58:34.804Z01/30 Write block 109 len 20480 data: 6 8 8 9 9
81952024-01-10T17:58:34.804Z02/30 Write block 248 len 36864 data: 6 6 5 5 6 6 4 5 6
81962024-01-10T17:58:34.805Z03/30 Write block 503 len 24576 data: 11 9 8 8 9 5
81972024-01-10T17:58:34.805Z04/30 Write block 540 len 8192 data: 7 7
81982024-01-10T17:58:34.805Z05/30 Read block 574 len 24576
81992024-01-10T17:58:34.805Z06/30 Write block 112 len 20480 data: 10 10 7 6 6
82002024-01-10T17:58:34.805Z07/30 Write block 404 len 24576 data: 7 9 10 10 8 7
82012024-01-10T17:58:34.805Z08/30 Write block 77 len 28672 data: 6 4 7 6 7 6 8
82022024-01-10T17:58:34.805Z09/30 Read block 280 len 32768
82032024-01-10T17:58:34.805Z10/30 Read block 404 len 12288
82042024-01-10T17:58:34.805Z11/30 Read block 135 len 20480
82052024-01-10T17:58:34.805Z12/30 Write block 11 len 4096 data: 5
82062024-01-10T17:58:34.805Z13/30 Read block 83 len 20480
82072024-01-10T17:58:34.805Z14/30 Write block 153 len 8192 data: 4 5
82082024-01-10T17:58:34.805Z15/30 Write block 164 len 28672 data: 11 12 12 11 9 9 7
82092024-01-10T17:58:34.806Z16/30 Read block 39 len 24576
82102024-01-10T17:58:34.806Z17/30 Read block 331 len 28672
82112024-01-10T17:58:34.806Z18/30 Flush
82122024-01-10T17:58:34.806Z19/30 Write block 381 len 8192 data: 5 5
82132024-01-10T17:58:34.806Z20/30 Write block 489 len 36864 data: 6 8 9 10 11 11 14 13 11
82142024-01-10T17:58:34.806Z21/30 Flush
82152024-01-10T17:58:34.806Z22/30 Write block 551 len 8192 data: 8 11
82162024-01-10T17:58:34.806Z23/30 Write block 401 len 28672 data: 6 5 7 8 10 11 11
82172024-01-10T17:58:34.806Z24/30 Write block 464 len 4096 data: 6
82182024-01-10T17:58:34.806Z25/30 Read block 85 len 36864
82192024-01-10T17:58:34.806Z26/30 Write block 50 len 12288 data: 6 5 5
82202024-01-10T17:58:34.807Z27/30 Write block 549 len 32768 data: 10 9 9 12 10 9 7 5
82212024-01-10T17:58:34.807Z28/30 Write block 427 len 24576 data: 5 6 6 4 4 4
82222024-01-10T17:58:34.807Z29/30 Write block 304 len 12288 data: 8 7 6
82232024-01-10T17:58:34.807Z30/30 Read block 269 len 12288
82242024-01-10T17:58:34.807Z----------------------------------------------------------------
82252024-01-10T17:58:34.807Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
82262024-01-10T17:58:34.807ZStates: Active Active Active
82272024-01-10T17:58:34.807ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82282024-01-10T17:58:34.807Z 1 Acked 1000 Write 5 Done Done Done false
82292024-01-10T17:58:34.807Z 2 Acked 1001 Write 9 Sent Done Done false
82302024-01-10T17:58:34.807Z 3 Acked 1002 Write 6 Sent Done Done false
82312024-01-10T17:58:34.807Z 4 Acked 1003 Write 2 Sent Done Done false
82322024-01-10T17:58:34.807Z 5 Acked 1004 Read 6 Sent Done Done false
82332024-01-10T17:58:34.822Z 6 Acked 1005 Write 5 Sent Done Done false
82342024-01-10T17:58:34.822Z 7 Acked 1006 Write 6 Sent Done Done false
82352024-01-10T17:58:34.822Z 8 Acked 1007 Write 7 Sent Done Done false
82362024-01-10T17:58:34.822Z 9 Acked 1008 Read 8 Sent Done Done false
82372024-01-10T17:58:34.822Z 10 Acked 1009 Read 3 Sent Done Done false
82382024-01-10T17:58:34.823Z 11 Acked 1010 Read 5 Sent Done Done false
82392024-01-10T17:58:34.823Z 12 Acked 1011 Write 1 Sent Done Done false
82402024-01-10T17:58:34.823Z 13 Acked 1012 Read 5 Sent Done Done false
82412024-01-10T17:58:34.823Z 14 Acked 1013 Write 2 Sent Done Done false
82422024-01-10T17:58:34.823Z 15 Acked 1014 Write 7 Sent Done Done false
82432024-01-10T17:58:34.823Z 16 Acked 1015 Read 6 Sent Done Done false
82442024-01-10T17:58:34.824Z 17 Acked 1016 Read 7 Sent Done Done false
82452024-01-10T17:58:34.824Z 18 Acked 1017 Flush 0 Sent Done Done false
82462024-01-10T17:58:34.824Z 19 Acked 1018 Write 2 Sent Done Done false
82472024-01-10T17:58:34.824Z 20 Acked 1019 Write 9 Sent Done Done false
82482024-01-10T17:58:34.824Z 21 Acked 1020 Flush 0 Sent Done Done false
82492024-01-10T17:58:34.824Z 22 Acked 1021 Write 2 Sent Done Done false
82502024-01-10T17:58:34.824Z 23 Acked 1022 Write 7 Sent Done Done false
82512024-01-10T17:58:34.824Z 24 Acked 1023 Write 1 Sent Done Done false
82522024-01-10T17:58:34.824Z 25 Acked 1024 Read 9 Sent Done Done false
82532024-01-10T17:58:34.825Z 26 Acked 1025 Write 3 Sent Done Done false
82542024-01-10T17:58:34.825Z 27 Acked 1026 Write 8 Sent Done Done false
82552024-01-10T17:58:34.825Z 28 Acked 1027 Write 6 Sent Done Done false
82562024-01-10T17:58:34.828Z 29 Acked 1028 Write 3 Sent Done Done false
82572024-01-10T17:58:34.828Z 30 Acked 1029 Read 3 Sent Done Done false
82582024-01-10T17:58:34.828Z STATES DS:0 DS:1 DS:2 TOTAL
82592024-01-10T17:58:34.828Z New 0 0 0 0
82602024-01-10T17:58:34.828Z Sent 29 0 0 29
82612024-01-10T17:58:34.828Z Done 1 30 30 61
82622024-01-10T17:58:34.828Z Skipped 0 0 0 0
82632024-01-10T17:58:34.829Z Error 0 0 0 0
82642024-01-10T17:58:34.829ZLast Flush: 0 1020 1020
82652024-01-10T17:58:34.829ZDownstairs last five completed:
82662024-01-10T17:58:34.829ZUpstairs last five completed: 30 29 28 27 26
82672024-01-10T17:58:34.829ZWrote out file "/tmp/test_repair_verify.data"
82682024-01-10T17:58:34.829Z
82692024-01-10T17:58:34.829ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
82702024-01-10T17:58:34.829ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
82712024-01-10T17:58:34.830Z 0 000-019 67 76 76 129 140 140 F F F <---
82722024-01-10T17:58:34.830Z 1 020-039 73 73 73 136 136 136 F F F
82732024-01-10T17:58:34.830Z 2 040-059 74 74 74 138 138 138 F T T <---
82742024-01-10T17:58:34.830Z 3 060-079 74 76 76 138 140 140 F F F <---
82752024-01-10T17:58:34.830Z 4 080-099 74 76 76 138 140 140 F F F <---
82762024-01-10T17:58:34.830Z 5 100-119 72 76 76 135 140 140 T F F <---
82772024-01-10T17:58:34.830Z 6 120-139 72 72 72 135 135 135 F F F
82782024-01-10T17:58:34.831Z 7 140-159 75 76 76 139 140 140 F F F <---
82792024-01-10T17:58:34.831Z 8 160-179 72 76 76 135 140 140 F F F <---
82802024-01-10T17:58:34.831Z 9 180-199 72 72 72 135 135 135 F F F
82812024-01-10T17:58:34.832Z 10 200-219 70 70 70 133 133 133 F F F
82822024-01-10T17:58:34.832Z 11 220-239 67 67 67 129 129 129 F F F
82832024-01-10T17:58:34.832Z 12 240-259 67 76 76 129 140 140 F F F <---
82842024-01-10T17:58:34.832Z 13 260-279 75 75 75 139 139 139 F F F
82852024-01-10T17:58:34.832Z 14 280-299 72 72 72 135 135 135 F F F
82862024-01-10T17:58:34.832Z 15 300-319 75 75 75 139 139 139 F T T <---
82872024-01-10T17:58:34.832Z 16 320-339 75 75 75 139 139 139 F F F
82882024-01-10T17:58:34.832Z 17 340-359 75 75 75 139 139 139 F F F
82892024-01-10T17:58:34.833Z 18 360-379 58 58 58 111 111 111 F F F
82902024-01-10T17:58:34.833Z 19 380-399 75 76 76 139 141 141 F F F <---
82912024-01-10T17:58:34.833Z 20 400-419 75 76 76 139 140 140 F T T <---
82922024-01-10T17:58:34.833Z 21 420-439 75 75 75 139 139 139 F T T <---
82932024-01-10T17:58:34.833Z 22 440-459 75 75 75 139 139 139 F F F
82942024-01-10T17:58:34.833Z 23 460-479 72 72 72 135 135 135 F T T <---
82952024-01-10T17:58:34.833Z 24 480-499 68 76 76 130 141 141 F F F <---
82962024-01-10T17:58:34.834Z 25 500-519 75 76 76 139 140 140 F F F <---
82972024-01-10T17:58:34.834Z 26 520-539 75 75 75 139 139 139 F F F
82982024-01-10T17:58:34.834Z 27 540-559 64 76 76 126 140 140 F T T <---
82992024-01-10T17:58:34.834Z 28 560-579 64 64 64 126 126 126 F F F
83002024-01-10T17:58:34.834Z 29 580-599 70 70 70 133 133 133 F F F
83012024-01-10T17:58:34.835ZMax gen: 76, Max flush: 141
83022024-01-10T17:58:34.836ZError: Difference in extent metadata found!
83032024-01-10T17:58:34.836ZA Difference in extent metadata is expected here
83042024-01-10T17:58:34.836ZOn loop 38
83052024-01-10T17:58:34.836Z
83062024-01-10T17:58:34.836ZVerifying data now
83072024-01-10T17:58:40.356ZNo change after verify
83082024-01-10T17:58:40.356ZLoop: 38 Downstairs dump after verify (and repair):
83092024-01-10T17:58:40.366ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83102024-01-10T17:58:40.366Z 0 000-019 76 76 76 140 140 140 F F F
83112024-01-10T17:58:40.366Z 1 020-039 73 73 73 136 136 136 F F F
83122024-01-10T17:58:40.366Z 2 040-059 77 77 77 142 142 142 F F F
83132024-01-10T17:58:40.366Z 3 060-079 76 76 76 140 140 140 F F F
83142024-01-10T17:58:40.366Z 4 080-099 76 76 76 140 140 140 F F F
83152024-01-10T17:58:40.366Z 5 100-119 76 76 76 140 140 140 F F F
83162024-01-10T17:58:40.366Z 6 120-139 72 72 72 135 135 135 F F F
83172024-01-10T17:58:40.366Z 7 140-159 76 76 76 140 140 140 F F F
83182024-01-10T17:58:40.367Z 8 160-179 76 76 76 140 140 140 F F F
83192024-01-10T17:58:40.367Z 9 180-199 72 72 72 135 135 135 F F F
83202024-01-10T17:58:40.367Z 10 200-219 70 70 70 133 133 133 F F F
83212024-01-10T17:58:40.367Z 11 220-239 67 67 67 129 129 129 F F F
83222024-01-10T17:58:40.367Z 12 240-259 76 76 76 140 140 140 F F F
83232024-01-10T17:58:40.367Z 13 260-279 75 75 75 139 139 139 F F F
83242024-01-10T17:58:40.367Z 14 280-299 72 72 72 135 135 135 F F F
83252024-01-10T17:58:40.367Z 15 300-319 77 77 77 142 142 142 F F F
83262024-01-10T17:58:40.367Z 16 320-339 75 75 75 139 139 139 F F F
83272024-01-10T17:58:40.367Z 17 340-359 75 75 75 139 139 139 F F F
83282024-01-10T17:58:40.367Z 18 360-379 58 58 58 111 111 111 F F F
83292024-01-10T17:58:40.368Z 19 380-399 76 76 76 141 141 141 F F F
83302024-01-10T17:58:40.368Z 20 400-419 77 77 77 142 142 142 F F F
83312024-01-10T17:58:40.368Z 21 420-439 77 77 77 142 142 142 F F F
83322024-01-10T17:58:40.368Z 22 440-459 75 75 75 139 139 139 F F F
83332024-01-10T17:58:40.368Z 23 460-479 77 77 77 142 142 142 F F F
83342024-01-10T17:58:40.368Z 24 480-499 76 76 76 141 141 141 F F F
83352024-01-10T17:58:40.368Z 25 500-519 76 76 76 140 140 140 F F F
83362024-01-10T17:58:40.368Z 26 520-539 75 75 75 139 139 139 F F F
83372024-01-10T17:58:40.368Z 27 540-559 77 77 77 142 142 142 F F F
83382024-01-10T17:58:40.368Z 28 560-579 64 64 64 126 126 126 F F F
83392024-01-10T17:58:40.368Z 29 580-599 70 70 70 133 133 133 F F F
83402024-01-10T17:58:40.368ZMax gen: 77, Max flush: 142
83412024-01-10T17:58:40.368Z
83422024-01-10T17:58:40.368ZBegin loop 39
83432024-01-10T17:58:40.370ZDownstairs to restart: 2
83442024-01-10T17:58:40.370Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.006379959Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467}
83452024-01-10T17:58:40.370Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007000368Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467}
83462024-01-10T17:58:40.370Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007067217Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467}
83472024-01-10T17:58:40.371Z{"msg":"Crucible 9f1619d0-33cd-4217-9f51-fef6940cb609 has session id: 98bf418e-15ac-4e4e-9b0d-44d64ea7de45","v":0,"name":"crucible","level":30Crucible runtime is spawned
83482024-01-10T17:58:40.371Z,"time":"2024-01-10T17:58:40.00710749Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83492024-01-10T17:58:40.371Z{"msg":"Upstairs opts: Upstairs UUID: 9f1619d0-33cd-4217-9f51-fef6940cb609, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007289685Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83502024-01-10T17:58:40.371Z{"msg":"Crucible stats registered with UUID: 9f1619d0-33cd-4217-9f51-fef6940cb609","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007328127Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83512024-01-10T17:58:40.371Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007370924Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467}
83522024-01-10T17:58:40.371Z{"msg":"9f1619d0-33cd-4217-9f51-fef6940cb609 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007506283Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83532024-01-10T17:58:40.371Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007564204Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"0","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83542024-01-10T17:58:40.371Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007590674Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"0","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83552024-01-10T17:58:40.371Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007627231Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"1","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83562024-01-10T17:58:40.371Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007673399Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"1","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83572024-01-10T17:58:40.371Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007720753Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"2","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83582024-01-10T17:58:40.371Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007798029Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"2","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83592024-01-10T17:58:40.371Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007841959Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83602024-01-10T17:58:40.372Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007884511Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83612024-01-10T17:58:40.372Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.007928384Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83622024-01-10T17:58:40.372Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.00796934Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83632024-01-10T17:58:40.372Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.008170878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83642024-01-10T17:58:40.373Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.008316015Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83652024-01-10T17:58:40.373Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.009284805Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"1","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83662024-01-10T17:58:40.373Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.009459915Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"1","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83672024-01-10T17:58:40.373Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.009599948Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"0","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83682024-01-10T17:58:40.373Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.009736038Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"0","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83692024-01-10T17:58:40.373Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.009807142Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"2","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83702024-01-10T17:58:40.373Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.009920857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"2","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83712024-01-10T17:58:40.373Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.010403552Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"1","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83722024-01-10T17:58:40.373Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.010651718Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"0","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83732024-01-10T17:58:40.373Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.010951648Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"2","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83742024-01-10T17:58:40.373Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011396655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"1","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83752024-01-10T17:58:40.373Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011490403Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"1","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83762024-01-10T17:58:40.373Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011537066Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83772024-01-10T17:58:40.373Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011604681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"0","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83782024-01-10T17:58:40.373Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011651186Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"0","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83792024-01-10T17:58:40.374Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011722468Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83802024-01-10T17:58:40.374Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011763283Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"2","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83812024-01-10T17:58:40.374Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011836202Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"2","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83822024-01-10T17:58:40.374Z{"msg":"[0]R flush_numbers[0..12]: [140, 136, 142, 140, 140, 140, 135, 140, 140, 135, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011869551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83832024-01-10T17:58:40.374Z{"msg":"[0]R generation[0..12]: [76, 73, 77, 76, 76, 76, 72, 76, 76, 72, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011913665Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83842024-01-10T17:58:40.374Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011939429Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83852024-01-10T17:58:40.374Z{"msg":"[1]R flush_numbers[0..12]: [140, 136, 142, 140, 140, 140, 135, 140, 140, 135, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011963705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83862024-01-10T17:58:40.374Z{"msg":"[1]R generation[0..12]: [76, 73, 77, 76, 76, 76, 72, 76, 76, 72, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.011986435Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83872024-01-10T17:58:40.374ZWait for a query_work_queue command to finish before sending IO
83882024-01-10T17:58:40.374Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012055588Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83892024-01-10T17:58:40.374Z{"msg":"[2]R flush_numbers[0..12]: [140, 136, 142, 140, 140, 140, 135, 140, 140, 135, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012092958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83902024-01-10T17:58:40.374Z{"msg":"[2]R generation[0..12]: [76, 73, 77, 76, 76, 76, 72, 76, 76, 72, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012150949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83912024-01-10T17:58:40.374Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.01219601Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83922024-01-10T17:58:40.374Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012239322Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83932024-01-10T17:58:40.375Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012269769Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83942024-01-10T17:58:40.375Z{"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012311642Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83952024-01-10T17:58:40.375Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012335044Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"":"downstairs","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83962024-01-10T17:58:40.376Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012356523Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83972024-01-10T17:58:40.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012387853Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"0","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83982024-01-10T17:58:40.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012429455Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"1","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
83992024-01-10T17:58:40.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012457896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"client":"2","":"downstairs","session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
84002024-01-10T17:58:40.376Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.01247977Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
84012024-01-10T17:58:40.376Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012531123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
84022024-01-10T17:58:40.376Z{"msg":"9f1619d0-33cd-4217-9f51-fef6940cb609 is now active with session: 98bf418e-15ac-4e4e-9b0d-44d64ea7de45","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012570365Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
84032024-01-10T17:58:40.376Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012632597Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467,"session_id":"98bf418e-15ac-4e4e-9b0d-44d64ea7de45"}
84042024-01-10T17:58:40.376Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:40.012672805Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1467}
84052024-01-10T17:58:40.378ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84062024-01-10T17:58:40.378ZLoading write count information from file "/tmp/test_repair_verify.data"
84072024-01-10T17:58:40.378ZRun Repair workload
84082024-01-10T17:58:40.378Z01/30 Write block 411 len 4096 data: 9
84092024-01-10T17:58:40.409Z02/30 Write block 113 len 8192 data: 11 8
84102024-01-10T17:58:40.409Z03/30 Read block 23 len 12288
84112024-01-10T17:58:40.409Z04/30 Read block 250 len 12288
84122024-01-10T17:58:40.409Z05/30 Write block 400 len 28672 data: 6 7 6 8 9 11 12
84132024-01-10T17:58:40.409Z06/30 Read block 352 len 16384
84142024-01-10T17:58:40.409Z07/30 Read block 591 len 16384
84152024-01-10T17:58:40.409Z08/30 Write block 339 len 40960 data: 7 6 6 9 9 7 7 6 7 6
84162024-01-10T17:58:40.410Z09/30 Read block 121 len 8192
84172024-01-10T17:58:40.410Z10/30 Read block 25 len 4096
84182024-01-10T17:58:40.410Z11/30 Write block 302 len 16384 data: 6 7 9 8
84192024-01-10T17:58:40.410Z12/30 Write block 174 len 40960 data: 6 5 5 7 7 6 8 7 7 7
84202024-01-10T17:58:40.410Z13/30 Read block 427 len 8192
84212024-01-10T17:58:40.410Z14/30 Read block 141 len 32768
84222024-01-10T17:58:40.410Z15/30 Write block 522 len 40960 data: 6 2 2 3 6 5 6 9 11 13
84232024-01-10T17:58:40.410Z16/30 Read block 120 len 36864
84242024-01-10T17:58:40.410Z17/30 Write block 94 len 40960 data: 6 9 9 9 7 8 7 6 6 5
84252024-01-10T17:58:40.410Z18/30 Read block 199 len 12288
84262024-01-10T17:58:40.410Z19/30 Read block 219 len 28672
84272024-01-10T17:58:40.410Z20/30 Flush
84282024-01-10T17:58:40.410Z21/30 Write block 547 len 24576 data: 10 10 11 10 10 13
84292024-01-10T17:58:40.410Z22/30 Write block 297 len 36864 data: 10 12 10 9 8 7 8 10 9
84302024-01-10T17:58:40.410Z23/30 Read block 351 len 20480
84312024-01-10T17:58:40.411Z24/30 Read block 173 len 32768
84322024-01-10T17:58:40.411Z25/30 Read block 192 len 8192
84332024-01-10T17:58:40.411Z26/30 Write block 452 len 24576 data: 6 7 6 4 4 5
84342024-01-10T17:58:40.411Z27/30 Write block 506 len 8192 data: 9 10
84352024-01-10T17:58:40.411Z28/30 Read block 32 len 36864
84362024-01-10T17:58:40.411Z29/30 Write block 274 len 8192 data: 11 12
84372024-01-10T17:58:40.411Z30/30 Read block 230 len 32768
84382024-01-10T17:58:40.411Z----------------------------------------------------------------
84392024-01-10T17:58:40.411Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
84402024-01-10T17:58:40.411ZStates: Active Active Active
84412024-01-10T17:58:40.411ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84422024-01-10T17:58:40.411Z 1 Acked 1000 Write 1 Done Done Done false
84432024-01-10T17:58:40.411Z 2 Acked 1001 Write 2 Done Done Done false
84442024-01-10T17:58:40.411Z 3 Acked 1002 Read 3 Done Done Done false
84452024-01-10T17:58:40.411Z 4 Acked 1003 Read 3 Done Done Done false
84462024-01-10T17:58:40.411Z 5 Acked 1004 Write 7 Done Done Done false
84472024-01-10T17:58:40.412Z 6 Acked 1005 Read 4 Done Done Done false
84482024-01-10T17:58:40.412Z 7 Acked 1006 Read 4 Done Done Done false
84492024-01-10T17:58:40.412Z 8 Acked 1007 Write 10 Done Done Sent false
84502024-01-10T17:58:40.412Z 9 Acked 1008 Read 2 Done Done Sent false
84512024-01-10T17:58:40.412Z 10 Acked 1009 Read 1 Done Done Sent false
84522024-01-10T17:58:40.412Z 11 Acked 1010 Write 4 Done Done Sent false
84532024-01-10T17:58:40.412Z 12 Acked 1011 Write 10 Done Done Sent false
84542024-01-10T17:58:40.412Z 13 Acked 1012 Read 2 Done Done Sent false
84552024-01-10T17:58:40.412Z 14 Acked 1013 Read 8 Done Done Sent false
84562024-01-10T17:58:40.412Z 15 Acked 1014 Write 10 Done Done Sent false
84572024-01-10T17:58:40.412Z 16 Acked 1015 Read 9 Done Done Sent false
84582024-01-10T17:58:40.412Z 17 Acked 1016 Write 10 Done Done Sent false
84592024-01-10T17:58:40.412Z 18 Acked 1017 Read 3 Done Done Sent false
84602024-01-10T17:58:40.412Z 19 Acked 1018 Read 7 Done Done Sent false
84612024-01-10T17:58:40.413Z 20 Acked 1019 Flush 0 Done Done Sent false
84622024-01-10T17:58:40.413Z 21 Acked 1020 Write 6 Done Done Sent false
84632024-01-10T17:58:40.413Z 22 Acked 1021 Write 9 Done Done Sent false
84642024-01-10T17:58:40.413Z 23 Acked 1022 Read 5 Done Done Sent false
84652024-01-10T17:58:40.413Z 24 Acked 1023 Read 8 Done Done Sent false
84662024-01-10T17:58:40.413Z 25 Acked 1024 Read 2 Done Done Sent false
84672024-01-10T17:58:40.413Z 26 Acked 1025 Write 6 Done Done Sent false
84682024-01-10T17:58:40.413Z 27 Acked 1026 Write 2 Done Done Sent false
84692024-01-10T17:58:40.413Z 28 Acked 1027 Read 9 Done Done Sent false
84702024-01-10T17:58:40.413Z 29 Acked 1028 Write 2 Done Done Sent false
84712024-01-10T17:58:40.413Z 30 Acked 1029 Read 8 Done Done Sent false
84722024-01-10T17:58:40.413Z STATES DS:0 DS:1 DS:2 TOTAL
84732024-01-10T17:58:40.413Z New 0 0 0 0
84742024-01-10T17:58:40.414Z Sent 0 0 23 23
84752024-01-10T17:58:40.420Z Done 30 30 7 67
84762024-01-10T17:58:40.421Z Skipped 0 0 0 0
84772024-01-10T17:58:40.421Z Error 0 0 0 0
84782024-01-10T17:58:40.421ZLast Flush: 1019 1019 0
84792024-01-10T17:58:40.421ZDownstairs last five completed:
84802024-01-10T17:58:40.421ZUpstairs last five completed: 30 29 28 27 26
84812024-01-10T17:58:40.421ZWrote out file "/tmp/test_repair_verify.data"
84822024-01-10T17:58:40.421Z
84832024-01-10T17:58:40.421ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
84842024-01-10T17:58:40.421ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
84852024-01-10T17:58:40.422Z 0 000-019 76 76 76 140 140 140 F F F
84862024-01-10T17:58:40.422Z 1 020-039 73 73 73 136 136 136 F F F
84872024-01-10T17:58:40.422Z 2 040-059 77 77 77 142 142 142 F F F
84882024-01-10T17:58:40.423Z 3 060-079 76 76 76 140 140 140 F F F
84892024-01-10T17:58:40.423Z 4 080-099 78 78 76 143 143 140 F F F <---
84902024-01-10T17:58:40.423Z 5 100-119 78 78 76 143 143 140 F F T <---
84912024-01-10T17:58:40.423Z 6 120-139 72 72 72 135 135 135 F F F
84922024-01-10T17:58:40.423Z 7 140-159 76 76 76 140 140 140 F F F
84932024-01-10T17:58:40.423Z 8 160-179 78 78 76 143 143 140 F F F <---
84942024-01-10T17:58:40.423Z 9 180-199 78 78 72 143 143 135 F F F <---
84952024-01-10T17:58:40.423Z 10 200-219 70 70 70 133 133 133 F F F
84962024-01-10T17:58:40.423Z 11 220-239 67 67 67 129 129 129 F F F
84972024-01-10T17:58:40.424Z 12 240-259 76 76 76 140 140 140 F F F
84982024-01-10T17:58:40.424Z 13 260-279 75 75 75 139 139 139 T T F <---
84992024-01-10T17:58:40.424Z 14 280-299 72 72 72 135 135 135 T T F <---
85002024-01-10T17:58:40.425Z 15 300-319 78 78 77 143 143 142 T T F <---
85012024-01-10T17:58:40.425Z 16 320-339 78 78 75 143 143 139 F F F <---
85022024-01-10T17:58:40.425Z 17 340-359 78 78 75 143 143 139 F F F <---
85032024-01-10T17:58:40.425Z 18 360-379 58 58 58 111 111 111 F F F
85042024-01-10T17:58:40.425Z 19 380-399 76 76 76 141 141 141 F F F
85052024-01-10T17:58:40.426Z 20 400-419 78 78 77 143 143 142 F F T <---
85062024-01-10T17:58:40.426Z 21 420-439 77 77 77 142 142 142 F F F
85072024-01-10T17:58:40.426Z 22 440-459 75 75 75 139 139 139 T T F <---
85082024-01-10T17:58:40.426Z 23 460-479 77 77 77 142 142 142 F F F
85092024-01-10T17:58:40.436Z 24 480-499 76 76 76 141 141 141 F F F
85102024-01-10T17:58:40.436Z 25 500-519 76 76 76 140 140 140 T T F <---
85112024-01-10T17:58:40.436Z 26 520-539 78 78 75 143 143 139 F F F <---
85122024-01-10T17:58:40.436Z 27 540-559 77 77 77 142 142 142 T T F <---
85132024-01-10T17:58:40.436Z 28 560-579 64 64 64 126 126 126 F F F
85142024-01-10T17:58:40.437Z 29 580-599 70 70 70 133 133 133 F F F
85152024-01-10T17:58:40.437ZMax gen: 78, Max flush: 143
85162024-01-10T17:58:40.437ZError: Difference in extent metadata found!
85172024-01-10T17:58:40.437ZA Difference in extent metadata is expected here
85182024-01-10T17:58:40.437ZOn loop 39
85192024-01-10T17:58:40.437Z
85202024-01-10T17:58:40.438ZVerifying data now
85212024-01-10T17:58:46.458ZNo change after verify
85222024-01-10T17:58:46.458ZLoop: 39 Downstairs dump after verify (and repair):
85232024-01-10T17:58:46.481ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85242024-01-10T17:58:46.481Z 0 000-019 76 76 76 140 140 140 F F F
85252024-01-10T17:58:46.481Z 1 020-039 73 73 73 136 136 136 F F F
85262024-01-10T17:58:46.481Z 2 040-059 77 77 77 142 142 142 F F F
85272024-01-10T17:58:46.481Z 3 060-079 76 76 76 140 140 140 F F F
85282024-01-10T17:58:46.481Z 4 080-099 78 78 78 143 143 143 F F F
85292024-01-10T17:58:46.484Z 5 100-119 78 78 78 143 143 143 F F F
85302024-01-10T17:58:46.484Z 6 120-139 72 72 72 135 135 135 F F F
85312024-01-10T17:58:46.484Z 7 140-159 76 76 76 140 140 140 F F F
85322024-01-10T17:58:46.484Z 8 160-179 78 78 78 143 143 143 F F F
85332024-01-10T17:58:46.484Z 9 180-199 78 78 78 143 143 143 F F F
85342024-01-10T17:58:46.484Z 10 200-219 70 70 70 133 133 133 F F F
85352024-01-10T17:58:46.484Z 11 220-239 67 67 67 129 129 129 F F F
85362024-01-10T17:58:46.484Z 12 240-259 76 76 76 140 140 140 F F F
85372024-01-10T17:58:46.485Z 13 260-279 79 79 79 144 144 144 F F F
85382024-01-10T17:58:46.485Z 14 280-299 79 79 79 144 144 144 F F F
85392024-01-10T17:58:46.485Z 15 300-319 79 79 79 144 144 144 F F F
85402024-01-10T17:58:46.485Z 16 320-339 78 78 78 143 143 143 F F F
85412024-01-10T17:58:46.485Z 17 340-359 78 78 78 143 143 143 F F F
85422024-01-10T17:58:46.485Z 18 360-379 58 58 58 111 111 111 F F F
85432024-01-10T17:58:46.485Z 19 380-399 76 76 76 141 141 141 F F F
85442024-01-10T17:58:46.485Z 20 400-419 78 78 78 143 143 143 F F F
85452024-01-10T17:58:46.486Z 21 420-439 77 77 77 142 142 142 F F F
85462024-01-10T17:58:46.486Z 22 440-459 79 79 79 144 144 144 F F F
85472024-01-10T17:58:46.486Z 23 460-479 77 77 77 142 142 142 F F F
85482024-01-10T17:58:46.486Z 24 480-499 76 76 76 141 141 141 F F F
85492024-01-10T17:58:46.486Z 25 500-519 79 79 79 144 144 144 F F F
85502024-01-10T17:58:46.486Z 26 520-539 78 78 78 143 143 143 F F F
85512024-01-10T17:58:46.486Z 27 540-559 79 79 79 144 144 144 F F F
85522024-01-10T17:58:46.486Z 28 560-579 64 64 64 126 126 126 F F F
85532024-01-10T17:58:46.486Z 29 580-599 70 70 70 133 133 133 F F F
85542024-01-10T17:58:46.486ZMax gen: 79, Max flush: 144
85552024-01-10T17:58:46.489Z
85562024-01-10T17:58:46.489ZBegin loop 40
85572024-01-10T17:58:46.489ZDownstairs to restart: 0
85582024-01-10T17:58:46.514Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.517006857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475}
85592024-01-10T17:58:46.514Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.51750855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475}
85602024-01-10T17:58:46.518Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.517583694Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475}
85612024-01-10T17:58:46.519Z{"msg":"Crucible 2911c8ed-e525-4557-9051-69b9f7c0a139 has session id: b6d7219c-d571-49e1-a35c-f0cb1cae8f8d","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.517671368Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85622024-01-10T17:58:46.519Z{"msg":"Crucible runtime is spawned
85632024-01-10T17:58:46.519ZUpstairs opts: Upstairs UUID: 2911c8ed-e525-4557-9051-69b9f7c0a139, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.517883786Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85642024-01-10T17:58:46.519Z{"msg":"Crucible stats registered with UUID: 2911c8ed-e525-4557-9051-69b9f7c0a139","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518000171Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85652024-01-10T17:58:46.519Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518089559Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475}
85662024-01-10T17:58:46.519Z{"msg":"2911c8ed-e525-4557-9051-69b9f7c0a139 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518190163Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85672024-01-10T17:58:46.519Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518276411Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"0","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85682024-01-10T17:58:46.519Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518318285Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"0","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85692024-01-10T17:58:46.519Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518375894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"1","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85702024-01-10T17:58:46.519Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518403445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"1","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85712024-01-10T17:58:46.519Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518446376Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"2","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85722024-01-10T17:58:46.520Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518534507Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"2","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85732024-01-10T17:58:46.520Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518623539Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85742024-01-10T17:58:46.520Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.5186727Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85752024-01-10T17:58:46.520Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.518715958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85762024-01-10T17:58:46.520Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.51876119Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85772024-01-10T17:58:46.520Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.519025879Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85782024-01-10T17:58:46.520Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.519154085Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85792024-01-10T17:58:46.520Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.520111481Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"2","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85802024-01-10T17:58:46.521Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.520196765Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"2","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85812024-01-10T17:58:46.522Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.52022659Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"1","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85822024-01-10T17:58:46.524Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.520288719Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"1","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85832024-01-10T17:58:46.524Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.520321426Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"0","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85842024-01-10T17:58:46.524Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.520363394Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"0","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85852024-01-10T17:58:46.529Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.521184705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"2","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85862024-01-10T17:58:46.529Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.521427421Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"0","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85872024-01-10T17:58:46.530Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.521598132Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"1","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85882024-01-10T17:58:46.530Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.521991864Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"2","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85892024-01-10T17:58:46.530Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522063262Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"2","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85902024-01-10T17:58:46.530Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522154659Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85912024-01-10T17:58:46.530Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522232583Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"0","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85922024-01-10T17:58:46.530Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522274238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"0","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85932024-01-10T17:58:46.530Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522312273Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85942024-01-10T17:58:46.530Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.52247732Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"1","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85952024-01-10T17:58:46.531Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522534231Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"1","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85962024-01-10T17:58:46.531Z{"msg":"[0]R flush_numbers[0..12]: [140, 136, 142, 140, 143, 143, 135, 140, 143, 143, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.52256071Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85972024-01-10T17:58:46.531Z{"msg":"[0]R generation[0..12]: [76, 73, 77, 76, 78, 78, 72, 76, 78, 78, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522602624Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85982024-01-10T17:58:46.531Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522627941Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
85992024-01-10T17:58:46.531Z{"msg":"[1]R flush_numbers[0..12]: [140, 136, 142, 140, 143, 143, 135, 140, 143, 143, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522669309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86002024-01-10T17:58:46.531Z{"msg":"[1]R generation[0..12]: [76, 73, 77, 76, 78, 78, 72, 76, 78, 78, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522693493Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86012024-01-10T17:58:46.531Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
86022024-01-10T17:58:46.531Z,"time":"2024-01-10T17:58:46.522734767Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86032024-01-10T17:58:46.531Z{"msg":"[2]R flush_numbers[0..12]: [140, 136, 142, 140, 143, 143, 135, 140, 143, 143, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522762907Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86042024-01-10T17:58:46.531Z{"msg":"[2]R generation[0..12]: [76, 73, 77, 76, 78, 78, 72, 76, 78, 78, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522803573Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86052024-01-10T17:58:46.531Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522828267Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86062024-01-10T17:58:46.531Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.52285147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86072024-01-10T17:58:46.531Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522873325Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86082024-01-10T17:58:46.531Z{"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.522978185Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86092024-01-10T17:58:46.531Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.523030807Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"":"downstairs","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86102024-01-10T17:58:46.531Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.523059138Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86112024-01-10T17:58:46.531Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.52311942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"0","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86122024-01-10T17:58:46.533Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.523146383Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"1","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86132024-01-10T17:58:46.533Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.523201606Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"client":"2","":"downstairs","session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86142024-01-10T17:58:46.533Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.52326789Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86152024-01-10T17:58:46.533Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.523304806Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86162024-01-10T17:58:46.533Z{"msg":"2911c8ed-e525-4557-9051-69b9f7c0a139 is now active with session: b6d7219c-d571-49e1-a35c-f0cb1cae8f8d","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.523362063Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86172024-01-10T17:58:46.533Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:46.523424299Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475,"session_id":"b6d7219c-d571-49e1-a35c-f0cb1cae8f8d"}
86182024-01-10T17:58:46.533Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86192024-01-10T17:58:46.533Z,"time":"2024-01-10T17:58:46.523504009Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1475}
86202024-01-10T17:58:46.533ZLoading write count information from file "/tmp/test_repair_verify.data"
86212024-01-10T17:58:46.533ZRun Repair workload
86222024-01-10T17:58:46.533Z01/30 Write block 373 len 16384 data: 6 5 5 5
86232024-01-10T17:58:46.533Z02/30 Read block 212 len 12288
86242024-01-10T17:58:46.533Z03/30 Write block 531 len 4096 data: 14
86252024-01-10T17:58:46.533Z04/30 Flush
86262024-01-10T17:58:46.537Z05/30 Write block 420 len 4096 data: 12
86272024-01-10T17:58:46.538Z06/30 Write block 525 len 12288 data: 4 7 6
86282024-01-10T17:58:46.538Z07/30 Write block 9 len 28672 data: 4 5 6 7 6 8 8
86292024-01-10T17:58:46.538Z08/30 Read block 229 len 28672
86302024-01-10T17:58:46.538Z09/30 Read block 374 len 40960
86312024-01-10T17:58:46.541Z10/30 Read block 521 len 4096
86322024-01-10T17:58:46.541Z11/30 Flush
86332024-01-10T17:58:46.544Z12/30 Flush
86342024-01-10T17:58:46.547Z13/30 Write block 48 len 28672 data: 5 7 7 6 6 5 8
86352024-01-10T17:58:46.547Z14/30 Flush
86362024-01-10T17:58:46.549Z15/30 Write block 195 len 20480 data: 10 7 7 8 8
86372024-01-10T17:58:46.550Z16/30 Read block 596 len 8192
86382024-01-10T17:58:46.553Z17/30 Flush
86392024-01-10T17:58:46.556Z18/30 Write block 42 len 36864 data: 6 5 5 6 4 3 6 8 8
86402024-01-10T17:58:46.556Z19/30 Read block 117 len 40960
86412024-01-10T17:58:46.559Z20/30 Read block 306 len 28672
86422024-01-10T17:58:46.562Z21/30 Read block 291 len 8192
86432024-01-10T17:58:46.564Z22/30 Read block 83 len 36864
86442024-01-10T17:58:46.567Z23/30 Read block 170 len 40960
86452024-01-10T17:58:46.570Z24/30 Write block 558 len 12288 data: 5 4 5
86462024-01-10T17:58:46.570Z25/30 Read block 513 len 8192
86472024-01-10T17:58:46.573Z26/30 Write block 7 len 24576 data: 5 4 5 6 7 8
86482024-01-10T17:58:46.573Z27/30 Write block 449 len 4096 data: 8
86492024-01-10T17:58:46.574Z28/30 Write block 0 len 24576 data: 4 5 6 7 8 6
86502024-01-10T17:58:46.577Z29/30 Read block 94 len 8192
86512024-01-10T17:58:46.580Z30/30 Write block 168 len 36864 data: 10 10 8 6 5 6 7 6 6
86522024-01-10T17:58:46.580Z----------------------------------------------------------------
86532024-01-10T17:58:46.580Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:26
86542024-01-10T17:58:46.580ZStates: Active Active Active
86552024-01-10T17:58:46.580ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
86562024-01-10T17:58:46.580Z 5 Acked 1004 Write 1 Done Done Done false
86572024-01-10T17:58:46.580Z 6 Acked 1005 Write 3 Done Done Done false
86582024-01-10T17:58:46.580Z 7 Acked 1006 Write 7 Done Done Done false
86592024-01-10T17:58:46.580Z 8 Acked 1007 Read 7 Done Done Done false
86602024-01-10T17:58:46.580Z 9 Acked 1008 Read 10 Sent Done Done false
86612024-01-10T17:58:46.580Z 10 Acked 1009 Read 1 Sent Done Done false
86622024-01-10T17:58:46.580Z 11 Acked 1010 Flush 0 Sent Done Done false
86632024-01-10T17:58:46.580Z 12 Acked 1011 Flush 0 Sent Done Done false
86642024-01-10T17:58:46.580Z 13 Acked 1012 Write 7 Sent Done Done false
86652024-01-10T17:58:46.580Z 14 Acked 1013 Flush 0 Sent Done Done false
86662024-01-10T17:58:46.581Z 15 Acked 1014 Write 5 Sent Done Done false
86672024-01-10T17:58:46.581Z 16 Acked 1015 Read 2 Sent Done Done false
86682024-01-10T17:58:46.581Z 17 Acked 1016 Flush 0 Sent Done Done false
86692024-01-10T17:58:46.581Z 18 Acked 1017 Write 9 Sent Done Done false
86702024-01-10T17:58:46.581Z 19 Acked 1018 Read 10 Sent Done Done false
86712024-01-10T17:58:46.581Z 20 Acked 1019 Read 7 Sent Done Done false
86722024-01-10T17:58:46.581Z 21 Acked 1020 Read 2 Sent Done Done false
86732024-01-10T17:58:46.581Z 22 Acked 1021 Read 9 Sent Done Done false
86742024-01-10T17:58:46.581Z 23 Acked 1022 Read 10 Sent Done Done false
86752024-01-10T17:58:46.581Z 24 Acked 1023 Write 3 Sent Done Done false
86762024-01-10T17:58:46.582Z 25 Acked 1024 Read 2 Sent Done Done false
86772024-01-10T17:58:46.582Z 26 Acked 1025 Write 6 Sent Done Done false
86782024-01-10T17:58:46.582Z 27 Acked 1026 Write 1 Sent Done Done false
86792024-01-10T17:58:46.582Z 28 Acked 1027 Write 6 Sent Done Done false
86802024-01-10T17:58:46.582Z 29 Acked 1028 Read 2 Sent Done Done false
86812024-01-10T17:58:46.582Z 30 Acked 1029 Write 9 Sent Sent Sent false
86822024-01-10T17:58:46.582Z STATES DS:0 DS:1 DS:2 TOTAL
86832024-01-10T17:58:46.582Z New 0 0 0 0
86842024-01-10T17:58:46.582Z Sent 22 1 1 24
86852024-01-10T17:58:46.582Z Done 4 25 25 54
86862024-01-10T17:58:46.582Z Skipped 0 0 0 0
86872024-01-10T17:58:46.582Z Error 0 0 0 0
86882024-01-10T17:58:46.582ZLast Flush: 1003 1016 1016
86892024-01-10T17:58:46.582ZDownstairs last five completed: 1003 1002 1001 1000
86902024-01-10T17:58:46.582ZUpstairs last five completed: 30 29 28 27 26
86912024-01-10T17:58:46.582ZWrote out file "/tmp/test_repair_verify.data"
86922024-01-10T17:58:46.586Z
86932024-01-10T17:58:46.589ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
86942024-01-10T17:58:46.614ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
86952024-01-10T17:58:46.614Z 0 000-019 76 80 80 140 146 146 T T T <---
86962024-01-10T17:58:46.614Z 1 020-039 73 73 73 136 136 136 F F F
86972024-01-10T17:58:46.614Z 2 040-059 77 80 80 142 148 148 F T T <---
86982024-01-10T17:58:46.614Z 3 060-079 76 76 76 140 140 140 F F F
86992024-01-10T17:58:46.614Z 4 080-099 78 78 78 143 143 143 F F F
87002024-01-10T17:58:46.623Z 5 100-119 78 78 78 143 143 143 F F F
87012024-01-10T17:58:46.623Z 6 120-139 72 72 72 135 135 135 F F F
87022024-01-10T17:58:46.623Z 7 140-159 76 76 76 140 140 140 F F F
87032024-01-10T17:58:46.623Z 8 160-179 78 78 78 143 143 143 F T T <---
87042024-01-10T17:58:46.623Z 9 180-199 78 80 80 143 149 149 F F F <---
87052024-01-10T17:58:46.623Z 10 200-219 70 70 70 133 133 133 F F F
87062024-01-10T17:58:46.623Z 11 220-239 67 67 67 129 129 129 F F F
87072024-01-10T17:58:46.623Z 12 240-259 76 76 76 140 140 140 F F F
87082024-01-10T17:58:46.623Z 13 260-279 79 79 79 144 144 144 F F F
87092024-01-10T17:58:46.623Z 14 280-299 79 79 79 144 144 144 F F F
87102024-01-10T17:58:46.624Z 15 300-319 79 79 79 144 144 144 F F F
87112024-01-10T17:58:46.624Z 16 320-339 78 78 78 143 143 143 F F F
87122024-01-10T17:58:46.624Z 17 340-359 78 78 78 143 143 143 F F F
87132024-01-10T17:58:46.624Z 18 360-379 80 80 80 145 145 145 F F F
87142024-01-10T17:58:46.624Z 19 380-399 76 76 76 141 141 141 F F F
87152024-01-10T17:58:46.624Z 20 400-419 78 78 78 143 143 143 F F F
87162024-01-10T17:58:46.624Z 21 420-439 77 80 80 142 146 146 T F F <---
87172024-01-10T17:58:46.624Z 22 440-459 79 79 79 144 144 144 F T T <---
87182024-01-10T17:58:46.624Z 23 460-479 77 77 77 142 142 142 F F F
87192024-01-10T17:58:46.624Z 24 480-499 76 76 76 141 141 141 F F F
87202024-01-10T17:58:46.624Z 25 500-519 79 79 79 144 144 144 F F F
87212024-01-10T17:58:46.624Z 26 520-539 80 80 80 145 146 146 T F F <---
87222024-01-10T17:58:46.625Z 27 540-559 79 79 79 144 144 144 F T T <---
87232024-01-10T17:58:46.625Z 28 560-579 64 64 64 126 126 126 F T T <---
87242024-01-10T17:58:46.625Z 29 580-599 70 70 70 133 133 133 F F F
87252024-01-10T17:58:46.625ZMax gen: 80, Max flush: 149
87262024-01-10T17:58:46.625ZError: Difference in extent metadata found!
87272024-01-10T17:58:46.625ZA Difference in extent metadata is expected here
87282024-01-10T17:58:46.625ZOn loop 40
87292024-01-10T17:58:46.625Z
87302024-01-10T17:58:46.632ZVerifying data now
87312024-01-10T17:58:51.597ZNo change after verify
87322024-01-10T17:58:51.597ZLoop: 40 Downstairs dump after verify (and repair):
87332024-01-10T17:58:51.620ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87342024-01-10T17:58:51.620Z 0 000-019 81 81 81 150 150 150 F F F
87352024-01-10T17:58:51.620Z 1 020-039 73 73 73 136 136 136 F F F
87362024-01-10T17:58:51.621Z 2 040-059 81 81 81 150 150 150 F F F
87372024-01-10T17:58:51.621Z 3 060-079 76 76 76 140 140 140 F F F
87382024-01-10T17:58:51.621Z 4 080-099 78 78 78 143 143 143 F F F
87392024-01-10T17:58:51.621Z 5 100-119 78 78 78 143 143 143 F F F
87402024-01-10T17:58:51.625Z 6 120-139 72 72 72 135 135 135 F F F
87412024-01-10T17:58:51.625Z 7 140-159 76 76 76 140 140 140 F F F
87422024-01-10T17:58:51.625Z 8 160-179 81 81 81 150 150 150 F F F
87432024-01-10T17:58:51.625Z 9 180-199 80 80 80 149 149 149 F F F
87442024-01-10T17:58:51.625Z 10 200-219 70 70 70 133 133 133 F F F
87452024-01-10T17:58:51.625Z 11 220-239 67 67 67 129 129 129 F F F
87462024-01-10T17:58:51.625Z 12 240-259 76 76 76 140 140 140 F F F
87472024-01-10T17:58:51.626Z 13 260-279 79 79 79 144 144 144 F F F
87482024-01-10T17:58:51.626Z 14 280-299 79 79 79 144 144 144 F F F
87492024-01-10T17:58:51.626Z 15 300-319 79 79 79 144 144 144 F F F
87502024-01-10T17:58:51.626Z 16 320-339 78 78 78 143 143 143 F F F
87512024-01-10T17:58:51.626Z 17 340-359 78 78 78 143 143 143 F F F
87522024-01-10T17:58:51.626Z 18 360-379 80 80 80 145 145 145 F F F
87532024-01-10T17:58:51.626Z 19 380-399 76 76 76 141 141 141 F F F
87542024-01-10T17:58:51.626Z 20 400-419 78 78 78 143 143 143 F F F
87552024-01-10T17:58:51.626Z 21 420-439 80 80 80 146 146 146 F F F
87562024-01-10T17:58:51.627Z 22 440-459 81 81 81 150 150 150 F F F
87572024-01-10T17:58:51.627Z 23 460-479 77 77 77 142 142 142 F F F
87582024-01-10T17:58:51.627Z 24 480-499 76 76 76 141 141 141 F F F
87592024-01-10T17:58:51.627Z 25 500-519 79 79 79 144 144 144 F F F
87602024-01-10T17:58:51.627Z 26 520-539 80 80 80 146 146 146 F F F
87612024-01-10T17:58:51.627Z 27 540-559 81 81 81 150 150 150 F F F
87622024-01-10T17:58:51.627Z 28 560-579 81 81 81 150 150 150 F F F
87632024-01-10T17:58:51.627Z 29 580-599 70 70 70 133 133 133 F F F
87642024-01-10T17:58:51.627ZMax gen: 81, Max flush: 150
87652024-01-10T17:58:51.627Z
87662024-01-10T17:58:51.630ZBegin loop 41
87672024-01-10T17:58:51.630ZDownstairs to restart: 0
87682024-01-10T17:58:51.651Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.653994277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483}
87692024-01-10T17:58:51.651Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.654459807Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483}
87702024-01-10T17:58:51.655Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.654487358Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483}
87712024-01-10T17:58:51.655Z{"msg":"Crucible 38d3fc8e-f5a3-4a3f-9042-6d7e3a055d27 has session id: bb8e76df-429c-4b84-ada2-02ec6838535c","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.654536466Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87722024-01-10T17:58:51.655Z{"msg":"Upstairs opts: Upstairs UUID: 38d3fc8e-f5a3-4a3f-9042-6d7e3a055d27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.654592674Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87732024-01-10T17:58:51.655Z{"msg":"Crucible stats registered with UUID: 38d3fc8e-f5a3-4a3f-9042-6d7e3a055d27","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.654666375Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87742024-01-10T17:58:51.655ZCrucible runtime is spawned
87752024-01-10T17:58:51.655Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.654998731Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483}
87762024-01-10T17:58:51.655Z{"msg":"38d3fc8e-f5a3-4a3f-9042-6d7e3a055d27 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.655292075Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87772024-01-10T17:58:51.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.6553411Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"0","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87782024-01-10T17:58:51.656Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.655366226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"0","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87792024-01-10T17:58:51.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.655388373Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"1","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87802024-01-10T17:58:51.656Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.655410886Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"1","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87812024-01-10T17:58:51.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.655432536Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"2","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87822024-01-10T17:58:51.656Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.655510721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"2","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87832024-01-10T17:58:51.656Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.655671618Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87842024-01-10T17:58:51.656Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.655769512Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87852024-01-10T17:58:51.656Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.65583097Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87862024-01-10T17:58:51.656Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.655913174Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87872024-01-10T17:58:51.656Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.65613663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87882024-01-10T17:58:51.656Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.656192493Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87892024-01-10T17:58:51.656Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.657302449Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"2","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87902024-01-10T17:58:51.656Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.657368232Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"2","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87912024-01-10T17:58:51.656Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.657410154Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"0","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87922024-01-10T17:58:51.656Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.657476945Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"0","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87932024-01-10T17:58:51.656Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.657503227Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"1","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87942024-01-10T17:58:51.656Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.657612871Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"1","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87952024-01-10T17:58:51.661Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.658324209Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"1","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87962024-01-10T17:58:51.661Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.658402406Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"2","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87972024-01-10T17:58:51.661Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.658767331Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"0","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87982024-01-10T17:58:51.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.659235236Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"1","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
87992024-01-10T17:58:51.661Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.659294445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"1","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88002024-01-10T17:58:51.661Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.659404044Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88012024-01-10T17:58:51.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.659476432Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"2","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88022024-01-10T17:58:51.661Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.659553626Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"2","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88032024-01-10T17:58:51.661Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.659625219Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88042024-01-10T17:58:51.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.659691033Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"0","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88052024-01-10T17:58:51.661Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.65976145Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"0","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88062024-01-10T17:58:51.661Z{"msg":"[0]R flush_numbers[0..12]: [150, 136, 150, 140, 143, 143, 135, 140, 150, 149, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.659833575Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88072024-01-10T17:58:51.661Z{"msg":"[0]R generation[0..12]: [81, 73, 81, 76, 78, 78, 72, 76, 81, 80, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.659878881Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88082024-01-10T17:58:51.661Z{"msg":"Wait for a query_work_queue command to finish before sending IO
88092024-01-10T17:58:51.662Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660034746Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88102024-01-10T17:58:51.662Z{"msg":"[1]R flush_numbers[0..12]: [150, 136, 150, 140, 143, 143, 135, 140, 150, 149, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.66011396Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88112024-01-10T17:58:51.662Z{"msg":"[1]R generation[0..12]: [81, 73, 81, 76, 78, 78, 72, 76, 81, 80, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660188971Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88122024-01-10T17:58:51.662Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660237547Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88132024-01-10T17:58:51.663Z{"msg":"[2]R flush_numbers[0..12]: [150, 136, 150, 140, 143, 143, 135, 140, 150, 149, 133, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.66031237Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88142024-01-10T17:58:51.663Z{"msg":"[2]R generation[0..12]: [81, 73, 81, 76, 78, 78, 72, 76, 81, 80, 70, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.6603815Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88152024-01-10T17:58:51.663Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660429504Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88162024-01-10T17:58:51.663Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660497705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88172024-01-10T17:58:51.663Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660541813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88182024-01-10T17:58:51.663Z{"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660609671Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88192024-01-10T17:58:51.663Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660680611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"":"downstairs","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88202024-01-10T17:58:51.663Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660755854Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88212024-01-10T17:58:51.663Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660799776Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"0","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88222024-01-10T17:58:51.663Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88232024-01-10T17:58:51.663Z,"time":"2024-01-10T17:58:51.660869177Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"1","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88242024-01-10T17:58:51.663Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660915161Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"client":"2","":"downstairs","session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88252024-01-10T17:58:51.663Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660966564Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88262024-01-10T17:58:51.663Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.660989383Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88272024-01-10T17:58:51.663Z{"msg":"38d3fc8e-f5a3-4a3f-9042-6d7e3a055d27 is now active with session: bb8e76df-429c-4b84-ada2-02ec6838535c","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.661010683Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88282024-01-10T17:58:51.663Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.661048585Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483,"session_id":"bb8e76df-429c-4b84-ada2-02ec6838535c"}
88292024-01-10T17:58:51.663Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:51.66107017Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1483}
88302024-01-10T17:58:51.663ZLoading write count information from file "/tmp/test_repair_verify.data"
88312024-01-10T17:58:51.663ZRun Repair workload
88322024-01-10T17:58:51.663Z01/30 Write block 142 len 20480 data: 7 7 7 5 6
88332024-01-10T17:58:51.667Z02/30 Write block 547 len 4096 data: 11
88342024-01-10T17:58:51.667Z03/30 Read block 285 len 4096
88352024-01-10T17:58:51.667Z04/30 Write block 30 len 16384 data: 12 10 10 9
88362024-01-10T17:58:51.667Z05/30 Read block 312 len 32768
88372024-01-10T17:58:51.670Z06/30 Write block 559 len 28672 data: 5 6 7 8 8 8 8
88382024-01-10T17:58:51.670Z07/30 Read block 488 len 28672
88392024-01-10T17:58:51.674Z08/30 Write block 195 len 40960 data: 11 8 8 9 9 8 7 8 7 5
88402024-01-10T17:58:51.674Z09/30 Read block 197 len 4096
88412024-01-10T17:58:51.677Z10/30 Read block 25 len 16384
88422024-01-10T17:58:51.677Z11/30 Flush
88432024-01-10T17:58:51.680Z12/30 Write block 195 len 28672 data: 12 9 9 10 10 9 8
88442024-01-10T17:58:51.683Z13/30 Read block 75 len 32768
88452024-01-10T17:58:51.686Z14/30 Write block 414 len 16384 data: 7 5 4 6
88462024-01-10T17:58:51.686Z15/30 Flush
88472024-01-10T17:58:51.689Z16/30 Write block 363 len 32768 data: 5 5 4 4 4 5 5 7
88482024-01-10T17:58:51.692Z17/30 Read block 25 len 32768
88492024-01-10T17:58:51.695Z18/30 Read block 259 len 16384
88502024-01-10T17:58:51.695Z19/30 Read block 4 len 28672
88512024-01-10T17:58:51.698Z20/30 Read block 418 len 16384
88522024-01-10T17:58:51.701Z21/30 Read block 239 len 28672
88532024-01-10T17:58:51.704Z22/30 Read block 558 len 8192
88542024-01-10T17:58:51.704Z23/30 Write block 383 len 32768 data: 4 4 4 4 5 4 3 4
88552024-01-10T17:58:51.707Z24/30 Read block 551 len 40960
88562024-01-10T17:58:51.710Z25/30 Read block 183 len 12288
88572024-01-10T17:58:51.710Z26/30 Write block 109 len 16384 data: 7 9 9 11
88582024-01-10T17:58:51.713Z27/30 Read block 337 len 4096
88592024-01-10T17:58:51.714Z28/30 Read block 243 len 4096
88602024-01-10T17:58:51.714Z29/30 Read block 247 len 4096
88612024-01-10T17:58:51.717Z30/30 Read block 384 len 36864
88622024-01-10T17:58:51.721Z----------------------------------------------------------------
88632024-01-10T17:58:51.721Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
88642024-01-10T17:58:51.721ZStates: Active Active Active
88652024-01-10T17:58:51.722ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
88662024-01-10T17:58:51.722Z 1 Acked 1000 Write 5 Sent Done Done false
88672024-01-10T17:58:51.722Z 2 Acked 1001 Write 1 Sent Done Done false
88682024-01-10T17:58:51.722Z 3 Acked 1002 Read 1 Sent Done Done false
88692024-01-10T17:58:51.722Z 4 Acked 1003 Write 4 Sent Done Done false
88702024-01-10T17:58:51.722Z 5 Acked 1004 Read 8 Sent Done Done false
88712024-01-10T17:58:51.722Z 6 Acked 1005 Write 7 Sent Done Done false
88722024-01-10T17:58:51.722Z 7 Acked 1006 Read 7 Sent Done Done false
88732024-01-10T17:58:51.722Z 8 Acked 1007 Write 10 Sent Done Done false
88742024-01-10T17:58:51.722Z 9 Acked 1008 Read 1 Sent Done Done false
88752024-01-10T17:58:51.722Z 10 Acked 1009 Read 4 Sent Done Done false
88762024-01-10T17:58:51.722Z 11 Acked 1010 Flush 0 Sent Done Done false
88772024-01-10T17:58:51.722Z 12 Acked 1011 Write 7 Sent Done Done false
88782024-01-10T17:58:51.722Z 13 Acked 1012 Read 8 Sent Done Done false
88792024-01-10T17:58:51.723Z 14 Acked 1013 Write 4 Sent Done Done false
88802024-01-10T17:58:51.723Z 15 Acked 1014 Flush 0 Sent Done Done false
88812024-01-10T17:58:51.723Z 16 Acked 1015 Write 8 Sent Done Done false
88822024-01-10T17:58:51.723Z 17 Acked 1016 Read 8 Sent Done Done false
88832024-01-10T17:58:51.723Z 18 Acked 1017 Read 4 Sent Done Done false
88842024-01-10T17:58:51.723Z 19 Acked 1018 Read 7 Sent Done Done false
88852024-01-10T17:58:51.723Z 20 Acked 1019 Read 4 Sent Done Done false
88862024-01-10T17:58:51.723Z 21 Acked 1020 Read 7 Sent Done Done false
88872024-01-10T17:58:51.723Z 22 Acked 1021 Read 2 Sent Done Done false
88882024-01-10T17:58:51.723Z 23 Acked 1022 Write 8 Sent Done Done false
88892024-01-10T17:58:51.723Z 24 Acked 1023 Read 10 Sent Done Done false
88902024-01-10T17:58:51.723Z 25 Acked 1024 Read 3 Sent Done Done false
88912024-01-10T17:58:51.723Z 26 Acked 1025 Write 4 Sent Done Done false
88922024-01-10T17:58:51.723Z 27 Acked 1026 Read 1 Sent Done Done false
88932024-01-10T17:58:51.723Z 28 Acked 1027 Read 1 Sent Done Done false
88942024-01-10T17:58:51.723Z 29 Acked 1028 Read 1 Sent Done Done false
88952024-01-10T17:58:51.723Z 30 Acked 1029 Read 9 Sent Done Done false
88962024-01-10T17:58:51.723Z STATES DS:0 DS:1 DS:2 TOTAL
88972024-01-10T17:58:51.723Z New 0 0 0 0
88982024-01-10T17:58:51.723Z Sent 30 0 0 30
88992024-01-10T17:58:51.723Z Done 0 30 30 60
89002024-01-10T17:58:51.723Z Skipped 0 0 0 0
89012024-01-10T17:58:51.723Z Error 0 0 0 0
89022024-01-10T17:58:51.723ZLast Flush: 0 1014 1014
89032024-01-10T17:58:51.724ZDownstairs last five completed:
89042024-01-10T17:58:51.724ZUpstairs last five completed: 30 29 28 27 26
89052024-01-10T17:58:51.724ZWrote out file "/tmp/test_repair_verify.data"
89062024-01-10T17:58:51.727Z
89072024-01-10T17:58:51.730ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89082024-01-10T17:58:51.754ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89092024-01-10T17:58:51.754Z 0 000-019 81 81 81 150 150 150 F F F
89102024-01-10T17:58:51.754Z 1 020-039 73 82 82 136 151 151 F F F <---
89112024-01-10T17:58:51.754Z 2 040-059 81 81 81 150 150 150 F F F
89122024-01-10T17:58:51.754Z 3 060-079 76 76 76 140 140 140 F F F
89132024-01-10T17:58:51.758Z 4 080-099 78 78 78 143 143 143 F F F
89142024-01-10T17:58:51.758Z 5 100-119 78 78 78 143 143 143 F T T <---
89152024-01-10T17:58:51.758Z 6 120-139 72 72 72 135 135 135 F F F
89162024-01-10T17:58:51.758Z 7 140-159 76 82 82 140 151 151 F F F <---
89172024-01-10T17:58:51.758Z 8 160-179 81 81 81 150 150 150 F F F
89182024-01-10T17:58:51.758Z 9 180-199 80 82 82 149 152 152 F F F <---
89192024-01-10T17:58:51.758Z 10 200-219 70 82 82 133 152 152 F F F <---
89202024-01-10T17:58:51.758Z 11 220-239 67 67 67 129 129 129 F F F
89212024-01-10T17:58:51.758Z 12 240-259 76 76 76 140 140 140 F F F
89222024-01-10T17:58:51.758Z 13 260-279 79 79 79 144 144 144 F F F
89232024-01-10T17:58:51.758Z 14 280-299 79 79 79 144 144 144 F F F
89242024-01-10T17:58:51.758Z 15 300-319 79 79 79 144 144 144 F F F
89252024-01-10T17:58:51.758Z 16 320-339 78 78 78 143 143 143 F F F
89262024-01-10T17:58:51.758Z 17 340-359 78 78 78 143 143 143 F F F
89272024-01-10T17:58:51.758Z 18 360-379 80 80 80 145 145 145 F T T <---
89282024-01-10T17:58:51.758Z 19 380-399 76 76 76 141 141 141 F T T <---
89292024-01-10T17:58:51.758Z 20 400-419 78 82 82 143 152 152 F F F <---
89302024-01-10T17:58:51.758Z 21 420-439 80 80 80 146 146 146 F F F
89312024-01-10T17:58:51.758Z 22 440-459 81 81 81 150 150 150 F F F
89322024-01-10T17:58:51.758Z 23 460-479 77 77 77 142 142 142 F F F
89332024-01-10T17:58:51.758Z 24 480-499 76 76 76 141 141 141 F F F
89342024-01-10T17:58:51.758Z 25 500-519 79 79 79 144 144 144 F F F
89352024-01-10T17:58:51.758Z 26 520-539 80 80 80 146 146 146 F F F
89362024-01-10T17:58:51.759Z 27 540-559 81 82 82 150 151 151 F F F <---
89372024-01-10T17:58:51.759Z 28 560-579 81 82 82 150 151 151 F F F <---
89382024-01-10T17:58:51.759Z 29 580-599 70 70 70 133 133 133 F F F
89392024-01-10T17:58:51.759ZMax gen: 82, Max flush: 152
89402024-01-10T17:58:51.759ZError: Difference in extent metadata found!
89412024-01-10T17:58:51.762ZA Difference in extent metadata is expected here
89422024-01-10T17:58:51.762ZOn loop 41
89432024-01-10T17:58:51.763Z
89442024-01-10T17:58:51.763ZVerifying data now
89452024-01-10T17:58:52.815ZNo change after verify
89462024-01-10T17:58:52.815ZLoop: 41 Downstairs dump after verify (and repair):
89472024-01-10T17:58:52.839ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89482024-01-10T17:58:52.839Z 0 000-019 81 81 81 150 150 150 F F F
89492024-01-10T17:58:52.839Z 1 020-039 82 82 82 151 151 151 F F F
89502024-01-10T17:58:52.839Z 2 040-059 81 81 81 150 150 150 F F F
89512024-01-10T17:58:52.840Z 3 060-079 76 76 76 140 140 140 F F F
89522024-01-10T17:58:52.840Z 4 080-099 78 78 78 143 143 143 F F F
89532024-01-10T17:58:52.840Z 5 100-119 83 83 83 153 153 153 F F F
89542024-01-10T17:58:52.840Z 6 120-139 72 72 72 135 135 135 F F F
89552024-01-10T17:58:52.840Z 7 140-159 82 82 82 151 151 151 F F F
89562024-01-10T17:58:52.840Z 8 160-179 81 81 81 150 150 150 F F F
89572024-01-10T17:58:52.840Z 9 180-199 82 82 82 152 152 152 F F F
89582024-01-10T17:58:52.841Z 10 200-219 82 82 82 152 152 152 F F F
89592024-01-10T17:58:52.841Z 11 220-239 67 67 67 129 129 129 F F F
89602024-01-10T17:58:52.841Z 12 240-259 76 76 76 140 140 140 F F F
89612024-01-10T17:58:52.841Z 13 260-279 79 79 79 144 144 144 F F F
89622024-01-10T17:58:52.841Z 14 280-299 79 79 79 144 144 144 F F F
89632024-01-10T17:58:53.261Z 15 300-319 79 79 79 144 144 144 F F F
89642024-01-10T17:58:53.261Z 16 320-339 78 78 78 143 143 143 F F F
89652024-01-10T17:58:53.261Z 17 340-359 78 78 78 143 143 143 F F F
89662024-01-10T17:58:53.261Z 18 360-379 83 83 83 153 153 153 F F F
89672024-01-10T17:58:53.261Z 19 380-399 83 83 83 153 153 153 F F F
89682024-01-10T17:58:53.261Z 20 400-419 82 82 82 152 152 152 F F F
89692024-01-10T17:58:53.261Z 21 420-439 80 80 80 146 146 146 F F F
89702024-01-10T17:58:53.261Z 22 440-459 81 81 81 150 150 150 F F F
89712024-01-10T17:58:53.261Z 23 460-479 77 77 77 142 142 142 F F F
89722024-01-10T17:58:53.261Z 24 480-499 76 76 76 141 141 141 F F F
89732024-01-10T17:58:53.262Z 25 500-519 79 79 79 144 144 144 F F F
89742024-01-10T17:58:53.262Z 26 520-539 80 80 80 146 146 146 F F F
89752024-01-10T17:58:53.262Z 27 540-559 82 82 82 151 151 151 F F F
89762024-01-10T17:58:53.262Z 28 560-579 82 82 82 151 151 151 F F F
89772024-01-10T17:58:53.262Z 29 580-599 70 70 70 133 133 133 F F F
89782024-01-10T17:58:53.262ZMax gen: 83, Max flush: 153
89792024-01-10T17:58:53.262Z
89802024-01-10T17:58:53.262ZBegin loop 42
89812024-01-10T17:58:53.262ZDownstairs to restart: 0
89822024-01-10T17:58:53.806Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.875857146Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491}
89832024-01-10T17:58:53.807Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.87644654Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491}
89842024-01-10T17:58:53.808Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.876475321Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491}
89852024-01-10T17:58:53.808Z{"msg":"Crucible 583dbd54-2bec-4aaa-b7a2-25bc62b58aa5 has session id: 2bc548ac-fcaa-4f29-8c6f-b6e67a999896","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.876507619Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89862024-01-10T17:58:53.808Z{"msg":"Upstairs opts: Upstairs UUID: 583dbd54-2bec-4aaa-b7a2-25bc62b58aa5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.876568095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89872024-01-10T17:58:53.808Z{"msg":"Crucible stats registered with UUID: 583dbd54-2bec-4aaa-b7a2-25bc62b58aa5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.876650025Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89882024-01-10T17:58:53.808ZCrucible runtime is spawned
89892024-01-10T17:58:53.808Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.876847507Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491}
89902024-01-10T17:58:53.808Z{"msg":"583dbd54-2bec-4aaa-b7a2-25bc62b58aa5 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.876992113Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89912024-01-10T17:58:53.808Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877054374Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"0","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89922024-01-10T17:58:53.808Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877080219Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"0","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89932024-01-10T17:58:53.808Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877102396Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"1","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89942024-01-10T17:58:53.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877125038Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"1","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89952024-01-10T17:58:53.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877148507Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"2","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89962024-01-10T17:58:53.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877219438Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"2","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89972024-01-10T17:58:53.809Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877289317Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89982024-01-10T17:58:53.809Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877316551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
89992024-01-10T17:58:53.809Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877347343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90002024-01-10T17:58:53.809Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877442424Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90012024-01-10T17:58:53.809Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877580837Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90022024-01-10T17:58:53.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.877783802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90032024-01-10T17:58:53.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.879007307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"2","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90042024-01-10T17:58:53.811Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.879084639Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"2","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90052024-01-10T17:58:53.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.879188436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"1","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90062024-01-10T17:58:53.811Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.879285553Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"1","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90072024-01-10T17:58:53.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.879999163Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"0","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90082024-01-10T17:58:53.811Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.880090776Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"0","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90092024-01-10T17:58:53.811Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.880205327Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"1","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90102024-01-10T17:58:53.811Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.880320464Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"2","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90112024-01-10T17:58:53.811Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.88101742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"0","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90122024-01-10T17:58:53.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.881346585Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"2","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90132024-01-10T17:58:53.812Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.881431739Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"2","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90142024-01-10T17:58:53.812Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.881505294Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90152024-01-10T17:58:53.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.881575655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"1","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90162024-01-10T17:58:53.812Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.881615097Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"1","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90172024-01-10T17:58:53.812Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.881665208Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90182024-01-10T17:58:53.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.881844855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"0","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90192024-01-10T17:58:53.812Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.881909077Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"0","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90202024-01-10T17:58:53.813Z{"msg":"[0]R flush_numbers[0..12]: [150, 151, 150, 140, 143, 153, 135, 151, 150, 152, 152, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.881938958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90212024-01-10T17:58:53.813Z{"msg":"[0]R generation[0..12]: [81, 82, 81, 76, 78, 83, 72, 82, 81, 82, 82, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.88200077Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90222024-01-10T17:58:53.813Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882033546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90232024-01-10T17:58:53.813Z{"msg":"Wait for a query_work_queue command to finish before sending IO
90242024-01-10T17:58:53.814Z[1]R flush_numbers[0..12]: [150, 151, 150, 140, 143, 153, 135, 151, 150, 152, 152, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882098582Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90252024-01-10T17:58:53.814Z{"msg":"[1]R generation[0..12]: [81, 82, 81, 76, 78, 83, 72, 82, 81, 82, 82, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882162553Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90262024-01-10T17:58:53.814Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.88219503Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90272024-01-10T17:58:53.814Z{"msg":"[2]R flush_numbers[0..12]: [150, 151, 150, 140, 143, 153, 135, 151, 150, 152, 152, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882253755Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90282024-01-10T17:58:53.814Z{"msg":"[2]R generation[0..12]: [81, 82, 81, 76, 78, 83, 72, 82, 81, 82, 82, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882284817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90292024-01-10T17:58:53.814Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882343594Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90302024-01-10T17:58:53.814Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882369659Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90312024-01-10T17:58:53.814Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.88239318Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90322024-01-10T17:58:53.814Z{"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882454636Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90332024-01-10T17:58:53.814Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882480881Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"":"downstairs","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90342024-01-10T17:58:53.816Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882539501Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90352024-01-10T17:58:53.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882564021Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"0","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90362024-01-10T17:58:53.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882624201Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"1","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90372024-01-10T17:58:53.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882651241Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"client":"2","":"downstairs","session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90382024-01-10T17:58:53.817Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882708871Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90392024-01-10T17:58:53.817Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882733877Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90402024-01-10T17:58:53.817Z{"msg":"583dbd54-2bec-4aaa-b7a2-25bc62b58aa5 is now active with session: 2bc548ac-fcaa-4f29-8c6f-b6e67a999896","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882788931Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90412024-01-10T17:58:53.817Z{"msgRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90422024-01-10T17:58:53.818Z":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882821919Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491,"session_id":"2bc548ac-fcaa-4f29-8c6f-b6e67a999896"}
90432024-01-10T17:58:53.818Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:52.882876322Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1491}
90442024-01-10T17:58:53.818ZLoading write count information from file "/tmp/test_repair_verify.data"
90452024-01-10T17:58:53.818ZRun Repair workload
90462024-01-10T17:58:53.818Z01/30 Write block 152 len 8192 data: 5 5
90472024-01-10T17:58:53.818Z02/30 Read block 371 len 8192
90482024-01-10T17:58:53.818Z03/30 Write block 411 len 40960 data: 10 8 8 8 6 5 7 7 11 13
90492024-01-10T17:58:53.818Z04/30 Read block 498 len 12288
90502024-01-10T17:58:53.819Z05/30 Write block 147 len 20480 data: 8 8 8 9 7
90512024-01-10T17:58:53.819Z06/30 Write block 87 len 4096 data: 5
90522024-01-10T17:58:53.819Z07/30 Write block 409 len 12288 data: 8 10 11
90532024-01-10T17:58:53.819Z08/30 Write block 363 len 8192 data: 6 6
90542024-01-10T17:58:53.819Z09/30 Write block 275 len 28672 data: 13 10 8 7 9 8 9
90552024-01-10T17:58:53.819Z10/30 Write block 310 len 4096 data: 3
90562024-01-10T17:58:53.819Z11/30 Read block 17 len 8192
90572024-01-10T17:58:53.820Z12/30 Read block 399 len 36864
90582024-01-10T17:58:53.820Z13/30 Read block 150 len 16384
90592024-01-10T17:58:53.820Z14/30 Read block 350 len 16384
90602024-01-10T17:58:53.820Z15/30 Flush
90612024-01-10T17:58:53.820Z16/30 Write block 127 len 40960 data: 4 3 3 6 5 5 5 5 6 4
90622024-01-10T17:58:53.820Z17/30 Write block 20 len 16384 data: 6 7 5 7
90632024-01-10T17:58:53.820Z18/30 Read block 368 len 16384
90642024-01-10T17:58:53.821Z19/30 Flush
90652024-01-10T17:58:53.821Z20/30 Write block 438 len 20480 data: 4 6 6 8 6
90662024-01-10T17:58:53.821Z21/30 Flush
90672024-01-10T17:58:53.821Z22/30 Flush
90682024-01-10T17:58:53.821Z23/30 Write block 346 len 8192 data: 7 8
90692024-01-10T17:58:53.821Z24/30 Flush
90702024-01-10T17:58:53.821Z25/30 Write block 483 len 16384 data: 3 6 5 6
90712024-01-10T17:58:53.822Z26/30 Read block 129 len 28672
90722024-01-10T17:58:53.822Z27/30 Write block 567 len 36864 data: 6 7 7 5 5 4 5 7 8
90732024-01-10T17:58:53.823Z28/30 Read block 88 len 24576
90742024-01-10T17:58:53.823Z29/30 Write block 195 len 24576 data: 13 10 10 11 11 10
90752024-01-10T17:58:53.823Z30/30 Write block 187 len 40960 data: 9 10 10 10 10 10 11 12 14 11
90762024-01-10T17:58:53.824Z----------------------------------------------------------------
90772024-01-10T17:58:53.824Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
90782024-01-10T17:58:53.824ZStates: Active Active Active
90792024-01-10T17:58:53.824ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
90802024-01-10T17:58:53.824Z 1 Acked 1000 Write 2 Sent Done Done false
90812024-01-10T17:58:53.824Z 2 Acked 1001 Read 2 Sent Done Done false
90822024-01-10T17:58:53.835Z 3 Acked 1002 Write 10 Sent Done Done false
90832024-01-10T17:58:53.835Z 4 Acked 1003 Read 3 Sent Done Done false
90842024-01-10T17:58:53.835Z 5 Acked 1004 Write 5 Sent Done Done false
90852024-01-10T17:58:53.835Z 6 Acked 1005 Write 1 Sent Done Done false
90862024-01-10T17:58:53.835Z 7 Acked 1006 Write 3 Sent Done Done false
90872024-01-10T17:58:53.835Z 8 Acked 1007 Write 2 Sent Done Done false
90882024-01-10T17:58:53.835Z 9 Acked 1008 Write 7 Sent Done Done false
90892024-01-10T17:58:53.835Z 10 Acked 1009 Write 1 Sent Done Done false
90902024-01-10T17:58:53.835Z 11 Acked 1010 Read 2 Sent Done Done false
90912024-01-10T17:58:53.835Z 12 Acked 1011 Read 9 Sent Done Done false
90922024-01-10T17:58:53.835Z 13 Acked 1012 Read 4 Sent Done Done false
90932024-01-10T17:58:53.835Z 14 Acked 1013 Read 4 Sent Done Done false
90942024-01-10T17:58:53.835Z 15 Acked 1014 Flush 0 Sent Done Done false
90952024-01-10T17:58:53.835Z 16 Acked 1015 Write 10 Sent Done Done false
90962024-01-10T17:58:53.835Z 17 Acked 1016 Write 4 Sent Done Done false
90972024-01-10T17:58:53.837Z 18 Acked 1017 Read 4 Sent Done Done false
90982024-01-10T17:58:53.837Z 19 Acked 1018 Flush 0 Sent Done Done false
90992024-01-10T17:58:53.837Z 20 Acked 1019 Write 5 Sent Done Done false
91002024-01-10T17:58:53.837Z 21 Acked 1020 Flush 0 Sent Done Done false
91012024-01-10T17:58:53.837Z 22 Acked 1021 Flush 0 Sent Done Done false
91022024-01-10T17:58:53.837Z 23 Acked 1022 Write 2 Sent Done Done false
91032024-01-10T17:58:53.837Z 24 Acked 1023 Flush 0 Sent Done Done false
91042024-01-10T17:58:53.837Z 25 Acked 1024 Write 4 Sent Done Done false
91052024-01-10T17:58:53.837Z 26 Acked 1025 Read 7 Sent Done Done false
91062024-01-10T17:58:53.837Z 27 Acked 1026 Write 9 Sent Done Done false
91072024-01-10T17:58:53.837Z 28 Acked 1027 Read 6 Sent Done Done false
91082024-01-10T17:58:53.837Z 29 Acked 1028 Write 6 Sent Done Done false
91092024-01-10T17:58:53.837Z 30 Acked 1029 Write 10 Sent Sent Sent false
91102024-01-10T17:58:53.837Z STATES DS:0 DS:1 DS:2 TOTAL
91112024-01-10T17:58:53.838Z New 0 0 0 0
91122024-01-10T17:58:53.838Z Sent 30 1 1 32
91132024-01-10T17:58:53.838Z Done 0 29 29 58
91142024-01-10T17:58:53.838Z Skipped 0 0 0 0
91152024-01-10T17:58:53.838Z Error 0 0 0 0
91162024-01-10T17:58:53.838ZLast Flush: 0 1023 1023
91172024-01-10T17:58:53.838ZDownstairs last five completed:
91182024-01-10T17:58:53.838ZUpstairs last five completed: 30 29 28 27 26
91192024-01-10T17:58:53.838ZWrote out file "/tmp/test_repair_verify.data"
91202024-01-10T17:58:53.840Z
91212024-01-10T17:58:53.840ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91222024-01-10T17:58:53.840ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91232024-01-10T17:58:53.840Z 0 000-019 81 81 81 150 150 150 F F F
91242024-01-10T17:58:53.840Z 1 020-039 82 84 84 151 155 155 F F F <---
91252024-01-10T17:58:53.840Z 2 040-059 81 81 81 150 150 150 F F F
91262024-01-10T17:58:53.840Z 3 060-079 76 76 76 140 140 140 F F F
91272024-01-10T17:58:53.840Z 4 080-099 78 84 84 143 154 154 F F F <---
91282024-01-10T17:58:53.840Z 5 100-119 83 83 83 153 153 153 F F F
91292024-01-10T17:58:53.840Z 6 120-139 72 84 84 135 155 155 F F F <---
91302024-01-10T17:58:53.840Z 7 140-159 82 84 84 151 154 154 F F F <---
91312024-01-10T17:58:53.840Z 8 160-179 81 81 81 150 150 150 F F F
91322024-01-10T17:58:53.840Z 9 180-199 82 82 82 152 152 152 F T T <---
91332024-01-10T17:58:53.840Z 10 200-219 82 82 82 152 152 152 F T T <---
91342024-01-10T17:58:53.840Z 11 220-239 67 67 67 129 129 129 F F F
91352024-01-10T17:58:53.840Z 12 240-259 76 76 76 140 140 140 F F F
91362024-01-10T17:58:53.840Z 13 260-279 79 84 84 144 154 154 F F F <---
91372024-01-10T17:58:53.840Z 14 280-299 79 84 84 144 154 154 F F F <---
91382024-01-10T17:58:53.840Z 15 300-319 79 84 84 144 154 154 F F F <---
91392024-01-10T17:58:53.841Z 16 320-339 78 78 78 143 143 143 F F F
91402024-01-10T17:58:53.841Z 17 340-359 78 84 84 143 158 158 F F F <---
91412024-01-10T17:58:53.841Z 18 360-379 83 84 84 153 154 154 F F F <---
91422024-01-10T17:58:53.841Z 19 380-399 83 83 83 153 153 153 F F F
91432024-01-10T17:58:53.841Z 20 400-419 82 84 84 152 154 154 F F F <---
91442024-01-10T17:58:53.841Z 21 420-439 80 84 84 146 156 156 F F F <---
91452024-01-10T17:58:53.841Z 22 440-459 81 84 84 150 156 156 F F F <---
91462024-01-10T17:58:53.841Z 23 460-479 77 77 77 142 142 142 F F F
91472024-01-10T17:58:53.841Z 24 480-499 76 76 76 141 141 141 F T T <---
91482024-01-10T17:58:53.841Z 25 500-519 79 79 79 144 144 144 F F F
91492024-01-10T17:58:53.841Z 26 520-539 80 80 80 146 146 146 F F F
91502024-01-10T17:58:53.841Z 27 540-559 82 82 82 151 151 151 F F F
91512024-01-10T17:58:53.841Z 28 560-579 82 82 82 151 151 151 F T T <---
91522024-01-10T17:58:53.841Z 29 580-599 70 70 70 133 133 133 F F F
91532024-01-10T17:58:53.841ZMax gen: 84, Max flush: 158
91542024-01-10T17:58:53.841ZError: Difference in extent metadata found!
91552024-01-10T17:58:53.841ZA Difference in extent metadata is expected here
91562024-01-10T17:58:53.841ZOn loop 42
91572024-01-10T17:58:53.841Z
91582024-01-10T17:58:53.841ZVerifying data now
91592024-01-10T17:58:58.502ZNo change after verify
91602024-01-10T17:58:58.502ZLoop: 42 Downstairs dump after verify (and repair):
91612024-01-10T17:58:58.525ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91622024-01-10T17:58:58.525Z 0 000-019 81 81 81 150 150 150 F F F
91632024-01-10T17:58:58.525Z 1 020-039 84 84 84 155 155 155 F F F
91642024-01-10T17:58:58.525Z 2 040-059 81 81 81 150 150 150 F F F
91652024-01-10T17:58:58.535Z 3 060-079 76 76 76 140 140 140 F F F
91662024-01-10T17:58:58.535Z 4 080-099 84 84 84 154 154 154 F F F
91672024-01-10T17:58:58.535Z 5 100-119 83 83 83 153 153 153 F F F
91682024-01-10T17:58:58.536Z 6 120-139 84 84 84 155 155 155 F F F
91692024-01-10T17:58:58.536Z 7 140-159 84 84 84 154 154 154 F F F
91702024-01-10T17:58:58.536Z 8 160-179 81 81 81 150 150 150 F F F
91712024-01-10T17:58:58.536Z 9 180-199 85 85 85 159 159 159 F F F
91722024-01-10T17:58:58.536Z 10 200-219 85 85 85 159 159 159 F F F
91732024-01-10T17:58:58.536Z 11 220-239 67 67 67 129 129 129 F F F
91742024-01-10T17:58:58.536Z 12 240-259 76 76 76 140 140 140 F F F
91752024-01-10T17:58:58.536Z 13 260-279 84 84 84 154 154 154 F F F
91762024-01-10T17:58:58.537Z 14 280-299 84 84 84 154 154 154 F F F
91772024-01-10T17:58:58.537Z 15 300-319 84 84 84 154 154 154 F F F
91782024-01-10T17:58:58.537Z 16 320-339 78 78 78 143 143 143 F F F
91792024-01-10T17:58:58.537Z 17 340-359 84 84 84 158 158 158 F F F
91802024-01-10T17:58:58.537Z 18 360-379 84 84 84 154 154 154 F F F
91812024-01-10T17:58:58.537Z 19 380-399 83 83 83 153 153 153 F F F
91822024-01-10T17:58:58.537Z 20 400-419 84 84 84 154 154 154 F F F
91832024-01-10T17:58:58.538Z 21 420-439 84 84 84 156 156 156 F F F
91842024-01-10T17:58:58.538Z 22 440-459 84 84 84 156 156 156 F F F
91852024-01-10T17:58:58.538Z 23 460-479 77 77 77 142 142 142 F F F
91862024-01-10T17:58:58.538Z 24 480-499 85 85 85 159 159 159 F F F
91872024-01-10T17:58:58.538Z 25 500-519 79 79 79 144 144 144 F F F
91882024-01-10T17:58:58.538Z 26 520-539 80 80 80 146 146 146 F F F
91892024-01-10T17:58:58.538Z 27 540-559 82 82 82 151 151 151 F F F
91902024-01-10T17:58:58.539Z 28 560-579 85 85 85 159 159 159 F F F
91912024-01-10T17:58:58.540Z 29 580-599 70 70 70 133 133 133 F F F
91922024-01-10T17:58:58.540ZMax gen: 85, Max flush: 159
91932024-01-10T17:58:58.540Z
91942024-01-10T17:58:58.540ZBegin loop 43
91952024-01-10T17:58:58.540ZDownstairs to restart: 0
91962024-01-10T17:58:58.552Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.551580654Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499}
91972024-01-10T17:58:58.561Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.552099656Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499}
91982024-01-10T17:58:58.561Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.552199608Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499}
91992024-01-10T17:58:58.562Z{"msg":"Crucible 029114b5-0eff-4741-b118-d9f926ea36ae has session id: 857a632c-60b0-4d67-b51a-59beae288639","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.552239441Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92002024-01-10T17:58:58.562Z{"msg":"Crucible runtime is spawned
92012024-01-10T17:58:58.562ZUpstairs opts: Upstairs UUID: 029114b5-0eff-4741-b118-d9f926ea36ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.552331515Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92022024-01-10T17:58:58.562Z{"msg":"Crucible stats registered with UUID: 029114b5-0eff-4741-b118-d9f926ea36ae","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.552440971Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92032024-01-10T17:58:58.562Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.5525098Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499}
92042024-01-10T17:58:58.562Z{"msg":"029114b5-0eff-4741-b118-d9f926ea36ae active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.552690879Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92052024-01-10T17:58:58.562Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.55282937Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"0","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92062024-01-10T17:58:58.563Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.552908182Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"0","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92072024-01-10T17:58:58.563Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.553012898Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"1","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92082024-01-10T17:58:58.563Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.553151126Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"1","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92092024-01-10T17:58:58.563Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.55321211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"2","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92102024-01-10T17:58:58.563Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.553257982Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"2","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92112024-01-10T17:58:58.564Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.553352253Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92122024-01-10T17:58:58.564Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.553670696Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92132024-01-10T17:58:58.564Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.553795022Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92142024-01-10T17:58:58.564Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.553895422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92152024-01-10T17:58:58.564Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.553940484Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92162024-01-10T17:58:58.565Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.554007759Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92172024-01-10T17:58:58.565Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.554580782Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"0","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92182024-01-10T17:58:58.566Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.554646662Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"0","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92192024-01-10T17:58:58.566Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.554672384Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"2","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92202024-01-10T17:58:58.566Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.554694692Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"2","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92212024-01-10T17:58:58.566Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.554757476Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"1","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92222024-01-10T17:58:58.566Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.554784655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"1","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92232024-01-10T17:58:58.566Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.555575964Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"1","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92242024-01-10T17:58:58.567Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.5558538Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"2","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92252024-01-10T17:58:58.567Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.555995912Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"0","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92262024-01-10T17:58:58.568Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.556427681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"1","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92272024-01-10T17:58:58.568Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.556514955Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"1","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92282024-01-10T17:58:58.568Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.556546861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92292024-01-10T17:58:58.568Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.556923522Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"2","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92302024-01-10T17:58:58.568Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.556994026Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"2","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92312024-01-10T17:58:58.568Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.55702006Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92322024-01-10T17:58:58.568Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557073076Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"0","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92332024-01-10T17:58:58.569Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557096865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"0","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92342024-01-10T17:58:58.569Z{"msg":"[0]R flush_numbers[0..12]: [150, 155, 150, 140, 154, 153, 155, 154, 150, 159, 159, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557120529Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92352024-01-10T17:58:58.569Z{"msg":"[0]R generation[0..12]: [81, 84, 81, 76, 84, 83, 84, 84, 81, 85, 85, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557161298Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92362024-01-10T17:58:58.569Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557186591Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92372024-01-10T17:58:58.569Z{"msg":"[1]R flush_numbers[0..12]: [150, 155, 150, 140, 154, 153, 155, 154, 150, 159, 159, 129]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557227616Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92382024-01-10T17:58:58.569Z{"msg":"[1]R generation[0..12]: [81, 84, 81, 76, 84, 83, 84, 84, 81, 85, 85, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557251989Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92392024-01-10T17:58:58.570Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557303783Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92402024-01-10T17:58:58.570Z{"msg":"[2]R flush_numbers[0..12]: [150, 155, 150, 140, 154, 153, 155, 154, 150, 159, 159, 129]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
92412024-01-10T17:58:58.570Zlevel":30,"time":"2024-01-10T17:58:58.557331317Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92422024-01-10T17:58:58.570Z{"msg":"[2]R generation[0..12]: [81, 84, 81, 76, 84, 83, 84, 84, 81, 85, 85, 67]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.55735435Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92432024-01-10T17:58:58.570Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557381016Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92442024-01-10T17:58:58.570Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557507165Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92452024-01-10T17:58:58.570Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557558819Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92462024-01-10T17:58:58.571Z{"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557581953Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92472024-01-10T17:58:58.572Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557624507Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"":"downstairs","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92482024-01-10T17:58:58.572Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557678825Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92492024-01-10T17:58:58.572Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557700072Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"0","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92502024-01-10T17:58:58.572Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557730234Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"1","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92512024-01-10T17:58:58.572Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557753192Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"client":"2","":"downstairs","session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92522024-01-10T17:58:58.572Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557777149Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92532024-01-10T17:58:58.572Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557798442Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92542024-01-10T17:58:58.573Z{"msg":"029114b5-0eff-4741-b118-d9f926ea36ae is now active with session: 857a632c-60b0-4d67-b51a-59beae288639","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557858548Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92552024-01-10T17:58:58.573Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557880334Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499,"session_id":"857a632c-60b0-4d67-b51a-59beae288639"}
92562024-01-10T17:58:58.573Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:58:58.557900751Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1499}
92572024-01-10T17:58:58.573ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
92582024-01-10T17:58:58.578ZLoading write count information from file "/tmp/test_repair_verify.data"
92592024-01-10T17:58:58.578ZRun Repair workload
92602024-01-10T17:58:58.578Z01/30 Write block 394 len 16384 data: 6 6 6 6
92612024-01-10T17:58:58.578Z02/30 Write block 125 len 12288 data: 5 4 5
92622024-01-10T17:58:58.578Z03/30 Read block 374 len 20480
92632024-01-10T17:58:58.578Z04/30 Read block 506 len 36864
92642024-01-10T17:58:58.578Z05/30 Write block 48 len 20480 data: 7 9 9 7 7
92652024-01-10T17:58:58.578Z06/30 Write block 259 len 12288 data: 7 5 5
92662024-01-10T17:58:58.578Z07/30 Read block 410 len 16384
92672024-01-10T17:58:58.578Z08/30 Read block 499 len 40960
92682024-01-10T17:58:58.579Z09/30 Write block 237 len 20480 data: 5 5 6 7 6
92692024-01-10T17:58:58.579Z10/30 Read block 584 len 20480
92702024-01-10T17:58:58.579Z11/30 Read block 547 len 36864
92712024-01-10T17:58:58.583Z12/30 Write block 151 len 20480 data: 8 6 6 6 6
92722024-01-10T17:58:58.583Z13/30 Read block 418 len 32768
92732024-01-10T17:58:58.583Z14/30 Read block 507 len 12288
92742024-01-10T17:58:58.586Z15/30 Write block 515 len 36864 data: 11 10 10 9 8 8 8 7 3
92752024-01-10T17:58:58.586Z16/30 Flush
92762024-01-10T17:58:58.591Z17/30 Flush
92772024-01-10T17:58:58.595Z18/30 Write block 89 len 4096 data: 5
92782024-01-10T17:58:58.595Z19/30 Write block 375 len 4096 data: 6
92792024-01-10T17:58:58.595Z20/30 Read block 344 len 28672
92802024-01-10T17:58:58.598Z21/30 Write block 581 len 36864 data: 7 7 8 6 5 7 6 6 6
92812024-01-10T17:58:58.599Z22/30 Read block 113 len 20480
92822024-01-10T17:58:58.606Z23/30 Read block 580 len 8192
92832024-01-10T17:58:58.607Z24/30 Read block 317 len 28672
92842024-01-10T17:58:58.607Z25/30 Flush
92852024-01-10T17:58:58.617Z26/30 Write block 99 len 8192 data: 9 8
92862024-01-10T17:58:58.617Z27/30 Write block 416 len 4096 data: 6
92872024-01-10T17:58:58.617Z28/30 Read block 252 len 28672
92882024-01-10T17:58:58.617Z29/30 Write block 506 len 40960 data: 10 11 6 9 9 10 11 12 13 12
92892024-01-10T17:58:58.622Z30/30 Read block 434 len 32768
92902024-01-10T17:58:58.622Z----------------------------------------------------------------
92912024-01-10T17:58:58.622Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
92922024-01-10T17:58:58.622ZStates: Active Active Active
92932024-01-10T17:58:58.622ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
92942024-01-10T17:58:58.622Z 1 Acked 1000 Write 4 Sent Done Done false
92952024-01-10T17:58:58.623Z 2 Acked 1001 Write 3 Sent Done Done false
92962024-01-10T17:58:58.623Z 3 Acked 1002 Read 5 Sent Done Done false
92972024-01-10T17:58:58.623Z 4 Acked 1003 Read 9 Sent Done Done false
92982024-01-10T17:58:58.623Z 5 Acked 1004 Write 5 Sent Done Done false
92992024-01-10T17:58:58.623Z 6 Acked 1005 Write 3 Sent Done Done false
93002024-01-10T17:58:58.623Z 7 Acked 1006 Read 4 Sent Done Done false
93012024-01-10T17:58:58.623Z 8 Acked 1007 Read 10 Sent Done Done false
93022024-01-10T17:58:58.623Z 9 Acked 1008 Write 5 Sent Done Done false
93032024-01-10T17:58:58.624Z 10 Acked 1009 Read 5 Sent Done Done false
93042024-01-10T17:58:58.624Z 11 Acked 1010 Read 9 Sent Done Done false
93052024-01-10T17:58:58.624Z 12 Acked 1011 Write 5 Sent Done Done false
93062024-01-10T17:58:58.625Z 13 Acked 1012 Read 8 Sent Done Done false
93072024-01-10T17:58:58.625Z 14 Acked 1013 Read 3 Sent Done Done false
93082024-01-10T17:58:58.625Z 15 Acked 1014 Write 9 Sent Done Done false
93092024-01-10T17:58:58.626Z 16 Acked 1015 Flush 0 Sent Done Done false
93102024-01-10T17:58:58.626Z 17 Acked 1016 Flush 0 Sent Done Done false
93112024-01-10T17:58:58.626Z 18 Acked 1017 Write 1 Sent Done Done false
93122024-01-10T17:58:58.626Z 19 Acked 1018 Write 1 Sent Done Done false
93132024-01-10T17:58:58.626Z 20 Acked 1019 Read 7 Sent Done Done false
93142024-01-10T17:58:58.626Z 21 Acked 1020 Write 9 Sent Done Done false
93152024-01-10T17:58:58.626Z 22 Acked 1021 Read 5 Sent Done Done false
93162024-01-10T17:58:58.626Z 23 Acked 1022 Read 2 Sent Done Done false
93172024-01-10T17:58:58.626Z 24 Acked 1023 Read 7 Sent Done Done false
93182024-01-10T17:58:58.627Z 25 Acked 1024 Flush 0 Sent Done Done false
93192024-01-10T17:58:58.627Z 26 Acked 1025 Write 2 Sent Done Done false
93202024-01-10T17:58:58.627Z 27 Acked 1026 Write 1 Sent Done Done false
93212024-01-10T17:58:58.627Z 28 Acked 1027 Read 7 Sent Done Done false
93222024-01-10T17:58:58.627Z 29 Acked 1028 Write 10 Sent Done Done false
93232024-01-10T17:58:58.627Z 30 Acked 1029 Read 8 Sent Done Done false
93242024-01-10T17:58:58.627Z STATES DS:0 DS:1 DS:2 TOTAL
93252024-01-10T17:58:58.627Z New 0 0 0 0
93262024-01-10T17:58:58.628Z Sent 30 0 0 30
93272024-01-10T17:58:58.628Z Done 0 30 30 60
93282024-01-10T17:58:58.628Z Skipped 0 0 0 0
93292024-01-10T17:58:58.628Z Error 0 0 0 0
93302024-01-10T17:58:58.630ZLast Flush: 0 1024 1024
93312024-01-10T17:58:58.630ZDownstairs last five completed:
93322024-01-10T17:58:58.630ZUpstairs last five completed: 30 29 28 27 26
93332024-01-10T17:58:58.630ZWrote out file "/tmp/test_repair_verify.data"
93342024-01-10T17:58:58.642Z
93352024-01-10T17:58:58.643ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93362024-01-10T17:58:58.654ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93372024-01-10T17:58:58.654Z 0 000-019 81 81 81 150 150 150 F F F
93382024-01-10T17:58:58.654Z 1 020-039 84 84 84 155 155 155 F F F
93392024-01-10T17:58:58.654Z 2 040-059 81 86 86 150 160 160 F F F <---
93402024-01-10T17:58:58.654Z 3 060-079 76 76 76 140 140 140 F F F
93412024-01-10T17:58:58.655Z 4 080-099 84 86 86 154 162 162 F T T <---
93422024-01-10T17:58:58.659Z 5 100-119 83 83 83 153 153 153 F T T <---
93432024-01-10T17:58:58.659Z 6 120-139 84 86 86 155 160 160 F F F <---
93442024-01-10T17:58:58.659Z 7 140-159 84 86 86 154 160 160 F F F <---
93452024-01-10T17:58:58.659Z 8 160-179 81 81 81 150 150 150 F F F
93462024-01-10T17:58:58.659Z 9 180-199 85 85 85 159 159 159 F F F
93472024-01-10T17:58:58.659Z 10 200-219 85 85 85 159 159 159 F F F
93482024-01-10T17:58:58.659Z 11 220-239 67 86 86 129 160 160 F F F <---
93492024-01-10T17:58:58.660Z 12 240-259 76 86 86 140 160 160 F F F <---
93502024-01-10T17:58:58.660Z 13 260-279 84 86 86 154 160 160 F F F <---
93512024-01-10T17:58:58.660Z 14 280-299 84 84 84 154 154 154 F F F
93522024-01-10T17:58:58.660Z 15 300-319 84 84 84 154 154 154 F F F
93532024-01-10T17:58:58.660Z 16 320-339 78 78 78 143 143 143 F F F
93542024-01-10T17:58:58.661Z 17 340-359 84 84 84 158 158 158 F F F
93552024-01-10T17:58:58.661Z 18 360-379 84 86 86 154 162 162 F F F <---
93562024-01-10T17:58:58.661Z 19 380-399 83 86 86 153 160 160 F F F <---
93572024-01-10T17:58:58.661Z 20 400-419 84 84 84 154 154 154 F T T <---
93582024-01-10T17:58:58.661Z 21 420-439 84 84 84 156 156 156 F F F
93592024-01-10T17:58:58.661Z 22 440-459 84 84 84 156 156 156 F F F
93602024-01-10T17:58:58.661Z 23 460-479 77 77 77 142 142 142 F F F
93612024-01-10T17:58:58.661Z 24 480-499 85 85 85 159 159 159 F F F
93622024-01-10T17:58:58.661Z 25 500-519 79 86 86 144 160 160 F T T <---
93632024-01-10T17:58:58.661Z 26 520-539 80 86 86 146 160 160 F F F <---
93642024-01-10T17:58:58.661Z 27 540-559 82 82 82 151 151 151 F F F
93652024-01-10T17:58:58.661Z 28 560-579 85 85 85 159 159 159 F F F
93662024-01-10T17:58:58.662Z 29 580-599 70 86 86 133 162 162 F F F <---
93672024-01-10T17:58:58.662ZMax gen: 86, Max flush: 162
93682024-01-10T17:58:58.662ZError: Difference in extent metadata found!
93692024-01-10T17:58:58.662ZA Difference in extent metadata is expected here
93702024-01-10T17:58:58.662ZOn loop 43
93712024-01-10T17:58:58.662Z
93722024-01-10T17:58:58.668ZVerifying data now
93732024-01-10T17:59:04.005ZNo change after verify
93742024-01-10T17:59:04.005ZLoop: 43 Downstairs dump after verify (and repair):
93752024-01-10T17:59:04.027ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93762024-01-10T17:59:04.027Z 0 000-019 81 81 81 150 150 150 F F F
93772024-01-10T17:59:04.027Z 1 020-039 84 84 84 155 155 155 F F F
93782024-01-10T17:59:04.027Z 2 040-059 86 86 86 160 160 160 F F F
93792024-01-10T17:59:04.031Z 3 060-079 76 76 76 140 140 140 F F F
93802024-01-10T17:59:04.031Z 4 080-099 87 87 87 163 163 163 F F F
93812024-01-10T17:59:04.031Z 5 100-119 87 87 87 163 163 163 F F F
93822024-01-10T17:59:04.031Z 6 120-139 86 86 86 160 160 160 F F F
93832024-01-10T17:59:04.031Z 7 140-159 86 86 86 160 160 160 F F F
93842024-01-10T17:59:04.031Z 8 160-179 81 81 81 150 150 150 F F F
93852024-01-10T17:59:04.031Z 9 180-199 85 85 85 159 159 159 F F F
93862024-01-10T17:59:04.031Z 10 200-219 85 85 85 159 159 159 F F F
93872024-01-10T17:59:04.031Z 11 220-239 86 86 86 160 160 160 F F F
93882024-01-10T17:59:04.031Z 12 240-259 86 86 86 160 160 160 F F F
93892024-01-10T17:59:04.031Z 13 260-279 86 86 86 160 160 160 F F F
93902024-01-10T17:59:04.031Z 14 280-299 84 84 84 154 154 154 F F F
93912024-01-10T17:59:04.031Z 15 300-319 84 84 84 154 154 154 F F F
93922024-01-10T17:59:04.031Z 16 320-339 78 78 78 143 143 143 F F F
93932024-01-10T17:59:04.031Z 17 340-359 84 84 84 158 158 158 F F F
93942024-01-10T17:59:04.031Z 18 360-379 86 86 86 162 162 162 F F F
93952024-01-10T17:59:04.032Z 19 380-399 86 86 86 160 160 160 F F F
93962024-01-10T17:59:04.032Z 20 400-419 87 87 87 163 163 163 F F F
93972024-01-10T17:59:04.032Z 21 420-439 84 84 84 156 156 156 F F F
93982024-01-10T17:59:04.032Z 22 440-459 84 84 84 156 156 156 F F F
93992024-01-10T17:59:04.032Z 23 460-479 77 77 77 142 142 142 F F F
94002024-01-10T17:59:04.032Z 24 480-499 85 85 85 159 159 159 F F F
94012024-01-10T17:59:04.033Z 25 500-519 87 87 87 163 163 163 F F F
94022024-01-10T17:59:04.033Z 26 520-539 86 86 86 160 160 160 F F F
94032024-01-10T17:59:04.033Z 27 540-559 82 82 82 151 151 151 F F F
94042024-01-10T17:59:04.033Z 28 560-579 85 85 85 159 159 159 F F F
94052024-01-10T17:59:04.033Z 29 580-599 86 86 86 162 162 162 F F F
94062024-01-10T17:59:04.033ZMax gen: 87, Max flush: 163
94072024-01-10T17:59:04.037Z
94082024-01-10T17:59:04.038ZBegin loop 44
94092024-01-10T17:59:04.038ZDownstairs to restart: 2
94102024-01-10T17:59:04.062Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
94112024-01-10T17:59:04.066Z,"time":"2024-01-10T17:59:04.064664891Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507}
94122024-01-10T17:59:04.066Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.065416226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507}
94132024-01-10T17:59:04.066Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.065501956Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507}
94142024-01-10T17:59:04.067Z{"msg":"Crucible 68ddc513-fa39-4f1f-86af-c698a4047269 has session id: ed0e77b3-3f05-4cfc-8458-37dadb42dcfc","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.065545357Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94152024-01-10T17:59:04.068Z{"msg":"Upstairs opts: Upstairs UUID: 68ddc513-fa39-4f1f-86af-c698a4047269, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.065631953Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94162024-01-10T17:59:04.068Z{"msg":"Crucible stats registered with UUID: 68ddc513-fa39-4f1f-86af-c698a4047269","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.065739849Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94172024-01-10T17:59:04.068Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.065785574Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507}
94182024-01-10T17:59:04.068Z{"msg":"68ddc513-fa39-4f1f-86af-c698a4047269 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.065824146Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94192024-01-10T17:59:04.068Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.065896172Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"0","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94202024-01-10T17:59:04.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.065941684Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"0","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94212024-01-10T17:59:04.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.065985687Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"1","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94222024-01-10T17:59:04.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.066062255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"1","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94232024-01-10T17:59:04.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.066137609Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"2","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94242024-01-10T17:59:04.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.066374573Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"2","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94252024-01-10T17:59:04.075Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.066536695Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94262024-01-10T17:59:04.075Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.066585183Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94272024-01-10T17:59:04.075Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.066688633Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94282024-01-10T17:59:04.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.066732135Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94292024-01-10T17:59:04.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.066804729Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94302024-01-10T17:59:04.076Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.066877626Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94312024-01-10T17:59:04.076Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.067283721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"0","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94322024-01-10T17:59:04.076Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.067368218Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"0","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94332024-01-10T17:59:04.076Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.067471673Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"1","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94342024-01-10T17:59:04.076Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.067563162Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"1","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94352024-01-10T17:59:04.078Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.068190315Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"0","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94362024-01-10T17:59:04.078Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.068268581Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"1","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94372024-01-10T17:59:04.078Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.068383842Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"2","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94382024-01-10T17:59:04.078Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.068461462Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"2","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94392024-01-10T17:59:04.083Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.069205437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"0","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94402024-01-10T17:59:04.083Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.069319918Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"0","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94412024-01-10T17:59:04.083Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.069357751Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94422024-01-10T17:59:04.083Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.069379701Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"1","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94432024-01-10T17:59:04.083Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.069402151Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"1","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94442024-01-10T17:59:04.083Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.069424314Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94452024-01-10T17:59:04.084Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.069768037Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"2","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94462024-01-10T17:59:04.085Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070522945Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"2","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94472024-01-10T17:59:04.085Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.07058894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"2","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94482024-01-10T17:59:04.085Z{"msg":"[0]R flush_numbers[0..12]: [150, 155, 160, 140, 163, 163, 160, 160, 150, 159, 159, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070620821Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94492024-01-10T17:59:04.085Z{"msg":"[0]R generation[0..12]: [81, 84, 86, 76, 87, 87, 86, 86, 81, 85, 85, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070665015Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94502024-01-10T17:59:04.085Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070690674Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94512024-01-10T17:59:04.085Z{"msg":"[1]R flush_numbers[0..12]: [150, 155, 160, 140, 163, 163, 160, 160, 150, 159, 159, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.07073277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94522024-01-10T17:59:04.085Z{"msg":"[1]R generation[0..12]: [81, 84, 86, 76, 87, 87, 86, 86, 81, 85, 85, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070756923Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94532024-01-10T17:59:04.085Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070781275Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94542024-01-10T17:59:04.085Z{"msg":"[2]R flush_numbers[0..12]: [150, 155, 160, 140, 163, 163, 160, 160, 150, 159, 159, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070804639Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94552024-01-10T17:59:04.085Z{"msg":"[2]R generation[0..12]: [81, 84, 86, 76, 87, 87, 86, 86, 81, 85, 85, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070828117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94562024-01-10T17:59:04.085Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
94572024-01-10T17:59:04.085Z,"time":"2024-01-10T17:59:04.070886294Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94582024-01-10T17:59:04.085Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070910222Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94592024-01-10T17:59:04.086Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070933519Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94602024-01-10T17:59:04.086Z{"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070954616Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94612024-01-10T17:59:04.086Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.070994264Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"":"downstairs","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94622024-01-10T17:59:04.086Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.071017542Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94632024-01-10T17:59:04.086Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.071055685Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"0","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94642024-01-10T17:59:04.087Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.071078547Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"1","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94652024-01-10T17:59:04.087Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.071100025Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"client":"2","":"downstairs","session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94662024-01-10T17:59:04.087Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.07112166Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94672024-01-10T17:59:04.087Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.071141354Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94682024-01-10T17:59:04.087Z{"msg":"68ddc513-fa39-4f1f-86af-c698a4047269 is now active with session: ed0e77b3-3f05-4cfc-8458-37dadb42dcfc","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.071162473Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94692024-01-10T17:59:04.087Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.071199505Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507,"session_id":"ed0e77b3-3f05-4cfc-8458-37dadb42dcfc"}
94702024-01-10T17:59:04.087Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:04.071220948Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1507}
94712024-01-10T17:59:04.087ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
94722024-01-10T17:59:04.087ZLoading write count information from file "/tmp/test_repair_verify.data"
94732024-01-10T17:59:04.087ZRun Repair workload
94742024-01-10T17:59:04.088Z01/30 Flush
94752024-01-10T17:59:04.088Z02/30 Write block 361 len 12288 data: 4 5 7
94762024-01-10T17:59:04.088Z03/30 Read block 570 len 28672
94772024-01-10T17:59:04.088Z04/30 Read block 55 len 12288
94782024-01-10T17:59:04.088Z05/30 Flush
94792024-01-10T17:59:04.088Z06/30 Flush
94802024-01-10T17:59:04.088Z07/30 Write block 374 len 8192 data: 6 7
94812024-01-10T17:59:04.088Z08/30 Read block 364 len 28672
94822024-01-10T17:59:04.092Z09/30 Read block 360 len 24576
94832024-01-10T17:59:04.092Z10/30 Write block 377 len 24576 data: 4 3 4 5 6 6
94842024-01-10T17:59:04.092Z11/30 Flush
94852024-01-10T17:59:04.092Z12/30 Write block 525 len 16384 data: 5 8 7 7
94862024-01-10T17:59:04.092Z13/30 Read block 388 len 8192
94872024-01-10T17:59:04.096Z14/30 Read block 368 len 20480
94882024-01-10T17:59:04.096Z15/30 Read block 257 len 40960
94892024-01-10T17:59:04.100Z16/30 Read block 71 len 40960
94902024-01-10T17:59:04.104Z17/30 Write block 389 len 28672 data: 4 5 6 6 5 7 7
94912024-01-10T17:59:04.104Z18/30 Read block 456 len 40960
94922024-01-10T17:59:04.109Z19/30 Read block 382 len 12288
94932024-01-10T17:59:04.523Z20/30 Write block 451 len 36864 data: 5 7 8 7 5 5 6 5 5
94942024-01-10T17:59:04.523Z21/30 Flush
94952024-01-10T17:59:04.523Z22/30 Write block 511 len 4096 data: 11
94962024-01-10T17:59:04.523Z23/30 Write block 450 len 32768 data: 8 6 8 9 8 6 6 7
94972024-01-10T17:59:04.523Z24/30 Write block 199 len 12288 data: 12 11 9
94982024-01-10T17:59:04.523Z25/30 Write block 21 len 20480 data: 8 6 8 6 6
94992024-01-10T17:59:04.523Z26/30 Read block 474 len 32768
95002024-01-10T17:59:04.524Z27/30 Read block 87 len 40960
95012024-01-10T17:59:04.524Z28/30 Read block 132 len 40960
95022024-01-10T17:59:04.524Z29/30 Write block 282 len 4096 data: 8
95032024-01-10T17:59:04.524Z30/30 Read block 241 len 40960
95042024-01-10T17:59:04.524Z----------------------------------------------------------------
95052024-01-10T17:59:04.525Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
95062024-01-10T17:59:04.525ZStates: Active Active Active
95072024-01-10T17:59:04.525ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95082024-01-10T17:59:04.525Z 1 Acked 1000 Flush 0 Done Done Sent false
95092024-01-10T17:59:04.525Z 2 Acked 1001 Write 3 Done Done Sent false
95102024-01-10T17:59:04.525Z 3 Acked 1002 Read 7 Done Done Sent false
95112024-01-10T17:59:04.525Z 4 Acked 1003 Read 3 Done Done Sent false
95122024-01-10T17:59:04.525Z 5 Acked 1004 Flush 0 Done Done Sent false
95132024-01-10T17:59:04.526Z 6 Acked 1005 Flush 0 Done Done Sent false
95142024-01-10T17:59:04.526Z 7 Acked 1006 Write 2 Done Done Sent false
95152024-01-10T17:59:04.526Z 8 Acked 1007 Read 7 Done Done Sent false
95162024-01-10T17:59:04.526Z 9 Acked 1008 Read 6 Done Done Sent false
95172024-01-10T17:59:04.526Z 10 Acked 1009 Write 6 Done Done Sent false
95182024-01-10T17:59:04.526Z 11 Acked 1010 Flush 0 Done Done Sent false
95192024-01-10T17:59:04.526Z 12 Acked 1011 Write 4 Done Done Sent false
95202024-01-10T17:59:04.526Z 13 Acked 1012 Read 2 Done Done Sent false
95212024-01-10T17:59:04.526Z 14 Acked 1013 Read 5 Done Done Sent false
95222024-01-10T17:59:04.526Z 15 Acked 1014 Read 10 Done Done Sent false
95232024-01-10T17:59:04.526Z 16 Acked 1015 Read 10 Done Done Sent false
95242024-01-10T17:59:04.526Z 17 Acked 1016 Write 7 Done Done Sent false
95252024-01-10T17:59:04.527Z 18 Acked 1017 Read 10 Done Done Sent false
95262024-01-10T17:59:04.527Z 19 Acked 1018 Read 3 Done Done Sent false
95272024-01-10T17:59:04.527Z 20 Acked 1019 Write 9 Done Done Sent false
95282024-01-10T17:59:04.529Z 21 Acked 1020 Flush 0 Done Done Sent false
95292024-01-10T17:59:04.529Z 22 Acked 1021 Write 1 Done Done Sent false
95302024-01-10T17:59:04.529Z 23 Acked 1022 Write 8 Done Done Sent false
95312024-01-10T17:59:04.529Z 24 Acked 1023 Write 3 Done Done Sent false
95322024-01-10T17:59:04.529Z 25 Acked 1024 Write 5 Done Done Sent false
95332024-01-10T17:59:04.529Z 26 Acked 1025 Read 8 Done Done Sent false
95342024-01-10T17:59:04.529Z 27 Acked 1026 Read 10 Done Done Sent false
95352024-01-10T17:59:04.529Z 28 Acked 1027 Read 10 Done Done Sent false
95362024-01-10T17:59:04.529Z 29 Acked 1028 Write 1 Done Done Sent false
95372024-01-10T17:59:04.529Z 30 Acked 1029 Read 10 Done Done Sent false
95382024-01-10T17:59:04.529Z STATES DS:0 DS:1 DS:2 TOTAL
95392024-01-10T17:59:04.529Z New 0 0 0 0
95402024-01-10T17:59:04.530Z Sent 0 0 30 30
95412024-01-10T17:59:04.530Z Done 30 30 0 60
95422024-01-10T17:59:04.530Z Skipped 0 0 0 0
95432024-01-10T17:59:04.530Z Error 0 0 0 0
95442024-01-10T17:59:04.530ZLast Flush: 1020 1020 0
95452024-01-10T17:59:04.530ZDownstairs last five completed:
95462024-01-10T17:59:04.530ZUpstairs last five completed: 30 29 28 27 26
95472024-01-10T17:59:04.530ZWrote out file "/tmp/test_repair_verify.data"
95482024-01-10T17:59:04.530Z
95492024-01-10T17:59:04.530ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
95502024-01-10T17:59:04.530ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95512024-01-10T17:59:04.530Z 0 000-019 81 81 81 150 150 150 F F F
95522024-01-10T17:59:04.531Z 1 020-039 84 84 84 155 155 155 T T F <---
95532024-01-10T17:59:04.531Z 2 040-059 86 86 86 160 160 160 F F F
95542024-01-10T17:59:04.531Z 3 060-079 76 76 76 140 140 140 F F F
95552024-01-10T17:59:04.531Z 4 080-099 87 87 87 163 163 163 F F F
95562024-01-10T17:59:04.531Z 5 100-119 87 87 87 163 163 163 F F F
95572024-01-10T17:59:04.531Z 6 120-139 86 86 86 160 160 160 F F F
95582024-01-10T17:59:04.531Z 7 140-159 86 86 86 160 160 160 F F F
95592024-01-10T17:59:04.532Z 8 160-179 81 81 81 150 150 150 F F F
95602024-01-10T17:59:04.532Z 9 180-199 85 85 85 159 159 159 T T F <---
95612024-01-10T17:59:04.532Z 10 200-219 85 85 85 159 159 159 T T F <---
95622024-01-10T17:59:04.532Z 11 220-239 86 86 86 160 160 160 F F F
95632024-01-10T17:59:04.532Z 12 240-259 86 86 86 160 160 160 F F F
95642024-01-10T17:59:04.544Z 13 260-279 86 86 86 160 160 160 F F F
95652024-01-10T17:59:04.544Z 14 280-299 84 84 84 154 154 154 T T F <---
95662024-01-10T17:59:04.544Z 15 300-319 84 84 84 154 154 154 F F F
95672024-01-10T17:59:04.544Z 16 320-339 78 78 78 143 143 143 F F F
95682024-01-10T17:59:04.545Z 17 340-359 84 84 84 158 158 158 F F F
95692024-01-10T17:59:04.545Z 18 360-379 88 88 86 167 167 162 F F F <---
95702024-01-10T17:59:04.545Z 19 380-399 88 88 86 168 168 160 F F F <---
95712024-01-10T17:59:04.545Z 20 400-419 87 87 87 163 163 163 F F F
95722024-01-10T17:59:04.545Z 21 420-439 84 84 84 156 156 156 F F F
95732024-01-10T17:59:04.545Z 22 440-459 88 88 84 168 168 156 T T F <---
95742024-01-10T17:59:04.545Z 23 460-479 77 77 77 142 142 142 F F F
95752024-01-10T17:59:04.545Z 24 480-499 85 85 85 159 159 159 F F F
95762024-01-10T17:59:04.545Z 25 500-519 87 87 87 163 163 163 T T F <---
95772024-01-10T17:59:04.545Z 26 520-539 88 88 86 168 168 160 F F F <---
95782024-01-10T17:59:04.545Z 27 540-559 82 82 82 151 151 151 F F F
95792024-01-10T17:59:04.545Z 28 560-579 85 85 85 159 159 159 F F F
95802024-01-10T17:59:04.545Z 29 580-599 86 86 86 162 162 162 F F F
95812024-01-10T17:59:04.545ZMax gen: 88, Max flush: 168
95822024-01-10T17:59:04.545ZError: Difference in extent metadata found!
95832024-01-10T17:59:04.545ZA Difference in extent metadata is expected here
95842024-01-10T17:59:04.545ZOn loop 44
95852024-01-10T17:59:04.545Z
95862024-01-10T17:59:04.545ZVerifying data now
95872024-01-10T17:59:09.152ZNo change after verify
95882024-01-10T17:59:09.153ZLoop: 44 Downstairs dump after verify (and repair):
95892024-01-10T17:59:09.174ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95902024-01-10T17:59:09.174Z 0 000-019 81 81 81 150 150 150 F F F
95912024-01-10T17:59:09.174Z 1 020-039 89 89 89 169 169 169 F F F
95922024-01-10T17:59:09.174Z 2 040-059 86 86 86 160 160 160 F F F
95932024-01-10T17:59:09.175Z 3 060-079 76 76 76 140 140 140 F F F
95942024-01-10T17:59:09.175Z 4 080-099 87 87 87 163 163 163 F F F
95952024-01-10T17:59:09.175Z 5 100-119 87 87 87 163 163 163 F F F
95962024-01-10T17:59:09.178Z 6 120-139 86 86 86 160 160 160 F F F
95972024-01-10T17:59:09.178Z 7 140-159 86 86 86 160 160 160 F F F
95982024-01-10T17:59:09.179Z 8 160-179 81 81 81 150 150 150 F F F
95992024-01-10T17:59:09.179Z 9 180-199 89 89 89 169 169 169 F F F
96002024-01-10T17:59:09.179Z 10 200-219 89 89 89 169 169 169 F F F
96012024-01-10T17:59:09.179Z 11 220-239 86 86 86 160 160 160 F F F
96022024-01-10T17:59:09.179Z 12 240-259 86 86 86 160 160 160 F F F
96032024-01-10T17:59:09.179Z 13 260-279 86 86 86 160 160 160 F F F
96042024-01-10T17:59:09.179Z 14 280-299 89 89 89 169 169 169 F F F
96052024-01-10T17:59:09.179Z 15 300-319 84 84 84 154 154 154 F F F
96062024-01-10T17:59:09.179Z 16 320-339 78 78 78 143 143 143 F F F
96072024-01-10T17:59:09.179Z 17 340-359 84 84 84 158 158 158 F F F
96082024-01-10T17:59:09.179Z 18 360-379 88 88 88 167 167 167 F F F
96092024-01-10T17:59:09.179Z 19 380-399 88 88 88 168 168 168 F F F
96102024-01-10T17:59:09.179Z 20 400-419 87 87 87 163 163 163 F F F
96112024-01-10T17:59:09.179Z 21 420-439 84 84 84 156 156 156 F F F
96122024-01-10T17:59:09.179Z 22 440-459 89 89 89 169 169 169 F F F
96132024-01-10T17:59:09.179Z 23 460-479 77 77 77 142 142 142 F F F
96142024-01-10T17:59:09.179Z 24 480-499 85 85 85 159 159 159 F F F
96152024-01-10T17:59:09.179Z 25 500-519 89 89 89 169 169 169 F F F
96162024-01-10T17:59:09.179Z 26 520-539 88 88 88 168 168 168 F F F
96172024-01-10T17:59:09.180Z 27 540-559 82 82 82 151 151 151 F F F
96182024-01-10T17:59:09.180Z 28 560-579 85 85 85 159 159 159 F F F
96192024-01-10T17:59:09.180Z 29 580-599 86 86 86 162 162 162 F F F
96202024-01-10T17:59:09.180ZMax gen: 89, Max flush: 169
96212024-01-10T17:59:09.183Z
96222024-01-10T17:59:09.183ZBegin loop 45
96232024-01-10T17:59:09.184ZDownstairs to restart: 0
96242024-01-10T17:59:09.208Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.211585083Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515}
96252024-01-10T17:59:09.208Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212123725Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515}
96262024-01-10T17:59:09.213Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.21218074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515}
96272024-01-10T17:59:09.213Z{"msg":"Crucible 67f8c986-2f62-4000-82fb-540515b7b7be has session id: 0712a0de-f3be-47c9-ab0c-4cb5454bd96b","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212223002Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96282024-01-10T17:59:09.213Z{"msg":"Crucible runtime is spawned
96292024-01-10T17:59:09.213ZUpstairs opts: Upstairs UUID: 67f8c986-2f62-4000-82fb-540515b7b7be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212290441Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96302024-01-10T17:59:09.213Z{"msg":"Crucible stats registered with UUID: 67f8c986-2f62-4000-82fb-540515b7b7be","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.21237914Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96312024-01-10T17:59:09.213Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212469213Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515}
96322024-01-10T17:59:09.213Z{"msg":"67f8c986-2f62-4000-82fb-540515b7b7be active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212662019Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96332024-01-10T17:59:09.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212718198Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"0","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96342024-01-10T17:59:09.214Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212762536Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"0","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96352024-01-10T17:59:09.215Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212848839Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"1","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96362024-01-10T17:59:09.215Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212890998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"1","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96372024-01-10T17:59:09.215Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212934207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"2","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96382024-01-10T17:59:09.215Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.212979351Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"2","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96392024-01-10T17:59:09.215Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.213019353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96402024-01-10T17:59:09.215Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.213058942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96412024-01-10T17:59:09.215Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.213149127Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96422024-01-10T17:59:09.215Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.213222027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96432024-01-10T17:59:09.215Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.213475641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96442024-01-10T17:59:09.215Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.213568567Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96452024-01-10T17:59:09.215Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.214506116Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"1","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96462024-01-10T17:59:09.216Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.214595598Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"1","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96472024-01-10T17:59:09.216Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.214641627Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"2","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96482024-01-10T17:59:09.216Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.214684482Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"2","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96492024-01-10T17:59:09.216Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.215239667Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"0","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96502024-01-10T17:59:09.216Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.215373043Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"0","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96512024-01-10T17:59:09.216Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.215599923Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"1","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96522024-01-10T17:59:09.221Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.215982192Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"2","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96532024-01-10T17:59:09.221Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.216457626Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"0","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96542024-01-10T17:59:09.230Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.216757634Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"1","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96552024-01-10T17:59:09.230Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.216815719Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"1","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96562024-01-10T17:59:09.230Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.216840955Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96572024-01-10T17:59:09.230Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.216974884Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"2","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96582024-01-10T17:59:09.230Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217044558Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"2","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96592024-01-10T17:59:09.230Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217088008Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96602024-01-10T17:59:09.230Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217214914Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"0","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96612024-01-10T17:59:09.230Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217272737Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"0","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96622024-01-10T17:59:09.230Z{"msg":"[0]R flush_numbers[0..12]: [150, 169, 160, 140, 163, 163, 160, 160, 150, 169, 169, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217367735Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96632024-01-10T17:59:09.231Z{"msg":"[0]R generation[0..12]: [81, 89, 86, 76, 87, 87, 86, 86, 81, 89, 89, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217422196Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96642024-01-10T17:59:09.231Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.21746802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96652024-01-10T17:59:09.231Z{"msg":"[1]R flush_numbers[0..12]: [150, 169, 160, 140, 163, 163, 160, 160, 150, 169, 169, 160]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
96662024-01-10T17:59:09.231Z,"time":"2024-01-10T17:59:09.217524535Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96672024-01-10T17:59:09.231Z{"msg":"[1]R generation[0..12]: [81, 89, 86, 76, 87, 87, 86, 86, 81, 89, 89, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217569208Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96682024-01-10T17:59:09.231Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217632692Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96692024-01-10T17:59:09.231Z{"msg":"[2]R flush_numbers[0..12]: [150, 169, 160, 140, 163, 163, 160, 160, 150, 169, 169, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217676696Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96702024-01-10T17:59:09.231Z{"msg":"[2]R generation[0..12]: [81, 89, 86, 76, 87, 87, 86, 86, 81, 89, 89, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217719955Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96712024-01-10T17:59:09.231Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217762106Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96722024-01-10T17:59:09.232Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.21782302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96732024-01-10T17:59:09.232Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217866568Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96742024-01-10T17:59:09.233Z{"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217926047Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96752024-01-10T17:59:09.233Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.217967231Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"":"downstairs","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96762024-01-10T17:59:09.233Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.218002769Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96772024-01-10T17:59:09.233Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.218038993Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"0","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96782024-01-10T17:59:09.233Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.218135934Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"1","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96792024-01-10T17:59:09.234Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.218175801Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"client":"2","":"downstairs","session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96802024-01-10T17:59:09.234Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.218242476Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96812024-01-10T17:59:09.234Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.21827724Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96822024-01-10T17:59:09.234Z{"msg":"67f8c986-2f62-4000-82fb-540515b7b7be is now active with session: 0712a0de-f3be-47c9-ab0c-4cb5454bd96b","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
96832024-01-10T17:59:09.234Z,"time":"2024-01-10T17:59:09.218311226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96842024-01-10T17:59:09.234Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.218415755Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515,"session_id":"0712a0de-f3be-47c9-ab0c-4cb5454bd96b"}
96852024-01-10T17:59:09.235Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:09.218442769Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1515}
96862024-01-10T17:59:09.235ZLoading write count information from file "/tmp/test_repair_verify.data"
96872024-01-10T17:59:09.235ZRun Repair workload
96882024-01-10T17:59:09.236Z01/30 Write block 197 len 8192 data: 11 12
96892024-01-10T17:59:09.236Z02/30 Write block 119 len 32768 data: 6 7 6 6 5 5 6 5
96902024-01-10T17:59:09.236Z03/30 Read block 269 len 4096
96912024-01-10T17:59:09.248Z04/30 Read block 434 len 16384
96922024-01-10T17:59:09.249Z05/30 Write block 48 len 12288 data: 8 10 10
96932024-01-10T17:59:09.249Z06/30 Write block 188 len 8192 data: 11 11
96942024-01-10T17:59:09.249Z07/30 Write block 261 len 28672 data: 6 4 5 5 4 6 8
96952024-01-10T17:59:09.249Z08/30 Flush
96962024-01-10T17:59:09.250Z09/30 Flush
96972024-01-10T17:59:09.250Z10/30 Write block 217 len 8192 data: 9 6
96982024-01-10T17:59:09.250Z11/30 Read block 458 len 40960
96992024-01-10T17:59:09.250Z12/30 Read block 93 len 24576
97002024-01-10T17:59:09.250Z13/30 Read block 150 len 36864
97012024-01-10T17:59:09.250Z14/30 Write block 421 len 24576 data: 11 10 10 8 6 5
97022024-01-10T17:59:09.683Z15/30 Write block 528 len 36864 data: 8 10 12 15 11 11 10 9 9
97032024-01-10T17:59:09.683Z16/30 Write block 488 len 36864 data: 6 7 9 10 11 12 12 15 14
97042024-01-10T17:59:09.683Z17/30 Flush
97052024-01-10T17:59:09.684Z18/30 Write block 338 len 40960 data: 7 8 7 7 10 10 8 8 8 9
97062024-01-10T17:59:09.684Z19/30 Read block 243 len 36864
97072024-01-10T17:59:09.684Z20/30 Read block 330 len 36864
97082024-01-10T17:59:09.684Z21/30 Write block 492 len 40960 data: 12 13 13 16 15 12 11 11 8 8
97092024-01-10T17:59:09.684Z22/30 Read block 471 len 8192
97102024-01-10T17:59:09.684Z23/30 Flush
97112024-01-10T17:59:09.684Z24/30 Write block 4 len 12288 data: 9 7 6
97122024-01-10T17:59:09.684Z25/30 Read block 492 len 16384
97132024-01-10T17:59:09.685Z26/30 Write block 292 len 8192 data: 11 9
97142024-01-10T17:59:09.685Z27/30 Read block 128 len 20480
97152024-01-10T17:59:09.685Z28/30 Write block 132 len 36864 data: 6 6 6 7 5 6 6 7 8
97162024-01-10T17:59:09.685Z29/30 Write block 55 len 36864 data: 8 7 6 8 8 8 7 6 6
97172024-01-10T17:59:09.685Z30/30 Read block 216 len 36864
97182024-01-10T17:59:09.685Z----------------------------------------------------------------
97192024-01-10T17:59:09.685Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
97202024-01-10T17:59:09.685ZStates: Active Active Active
97212024-01-10T17:59:09.685ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97222024-01-10T17:59:09.685Z 1 Acked 1000 Write 2 Done Done Done false
97232024-01-10T17:59:09.685Z 2 Acked 1001 Write 8 Done Done Done false
97242024-01-10T17:59:09.685Z 3 Acked 1002 Read 1 Done Done Done false
97252024-01-10T17:59:09.685Z 4 Acked 1003 Read 4 Sent Done Done false
97262024-01-10T17:59:09.686Z 5 Acked 1004 Write 3 Sent Done Done false
97272024-01-10T17:59:09.686Z 6 Acked 1005 Write 2 Sent Done Done false
97282024-01-10T17:59:09.686Z 7 Acked 1006 Write 7 Sent Done Done false
97292024-01-10T17:59:09.686Z 8 Acked 1007 Flush 0 Sent Done Done false
97302024-01-10T17:59:09.686Z 9 Acked 1008 Flush 0 Sent Done Done false
97312024-01-10T17:59:09.686Z 10 Acked 1009 Write 2 Sent Done Done false
97322024-01-10T17:59:09.686Z 11 Acked 1010 Read 10 Sent Done Done false
97332024-01-10T17:59:09.686Z 12 Acked 1011 Read 6 Sent Done Done false
97342024-01-10T17:59:09.686Z 13 Acked 1012 Read 9 Sent Done Done false
97352024-01-10T17:59:09.686Z 14 Acked 1013 Write 6 Sent Done Done false
97362024-01-10T17:59:09.686Z 15 Acked 1014 Write 9 Sent Done Done false
97372024-01-10T17:59:09.686Z 16 Acked 1015 Write 9 Sent Done Done false
97382024-01-10T17:59:09.687Z 17 Acked 1016 Flush 0 Sent Done Done false
97392024-01-10T17:59:09.687Z 18 Acked 1017 Write 10 Sent Done Done false
97402024-01-10T17:59:09.687Z 19 Acked 1018 Read 9 Sent Done Done false
97412024-01-10T17:59:09.687Z 20 Acked 1019 Read 9 Sent Done Done false
97422024-01-10T17:59:09.687Z 21 Acked 1020 Write 10 Sent Done Done false
97432024-01-10T17:59:09.687Z 22 Acked 1021 Read 2 Sent Done Done false
97442024-01-10T17:59:09.687Z 23 Acked 1022 Flush 0 Sent Done Done false
97452024-01-10T17:59:09.687Z 24 Acked 1023 Write 3 Sent Done Done false
97462024-01-10T17:59:09.687Z 25 Acked 1024 Read 4 Sent Done Done false
97472024-01-10T17:59:09.687Z 26 Acked 1025 Write 2 Sent Done Done false
97482024-01-10T17:59:09.687Z 27 Acked 1026 Read 5 Sent Done Done false
97492024-01-10T17:59:09.687Z 28 Acked 1027 Write 9 Sent Done Done false
97502024-01-10T17:59:09.687Z 29 Acked 1028 Write 9 Sent Done Done false
97512024-01-10T17:59:09.687Z 30 Acked 1029 Read 9 Sent Sent Done false
97522024-01-10T17:59:09.688Z STATES DS:0 DS:1 DS:2 TOTAL
97532024-01-10T17:59:09.688Z New 0 0 0 0
97542024-01-10T17:59:09.688Z Sent 27 1 0 28
97552024-01-10T17:59:09.688Z Done 3 29 30 62
97562024-01-10T17:59:09.688Z Skipped 0 0 0 0
97572024-01-10T17:59:09.688Z Error 0 0 0 0
97582024-01-10T17:59:09.689ZLast Flush: 0 1022 1022
97592024-01-10T17:59:09.689ZDownstairs last five completed:
97602024-01-10T17:59:09.689ZUpstairs last five completed: 30 29 28 27 26
97612024-01-10T17:59:09.689ZWrote out file "/tmp/test_repair_verify.data"
97622024-01-10T17:59:09.689Z
97632024-01-10T17:59:09.689ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
97642024-01-10T17:59:09.690ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97652024-01-10T17:59:09.690Z 0 000-019 81 81 81 150 150 150 F T T <---
97662024-01-10T17:59:09.690Z 1 020-039 89 89 89 169 169 169 F F F
97672024-01-10T17:59:09.690Z 2 040-059 86 90 90 160 170 170 F T T <---
97682024-01-10T17:59:09.690Z 3 060-079 76 76 76 140 140 140 F T T <---
97692024-01-10T17:59:09.690Z 4 080-099 87 87 87 163 163 163 F F F
97702024-01-10T17:59:09.690Z 5 100-119 87 90 90 163 170 170 T F F <---
97712024-01-10T17:59:09.690Z 6 120-139 86 90 90 160 170 170 T T T <---
97722024-01-10T17:59:09.690Z 7 140-159 86 86 86 160 160 160 F T T <---
97732024-01-10T17:59:09.690Z 8 160-179 81 81 81 150 150 150 F F F
97742024-01-10T17:59:09.693Z 9 180-199 89 90 90 169 170 170 T F F <---
97752024-01-10T17:59:09.693Z 10 200-219 89 90 90 169 172 172 F F F <---
97762024-01-10T17:59:09.693Z 11 220-239 86 86 86 160 160 160 F F F
97772024-01-10T17:59:09.693Z 12 240-259 86 86 86 160 160 160 F F F
97782024-01-10T17:59:09.693Z 13 260-279 86 90 90 160 170 170 F F F <---
97792024-01-10T17:59:09.693Z 14 280-299 89 89 89 169 169 169 F T T <---
97802024-01-10T17:59:09.693Z 15 300-319 84 84 84 154 154 154 F F F
97812024-01-10T17:59:09.693Z 16 320-339 78 90 90 143 173 173 F F F <---
97822024-01-10T17:59:09.694Z 17 340-359 84 90 90 158 173 173 F F F <---
97832024-01-10T17:59:09.694Z 18 360-379 88 88 88 167 167 167 F F F
97842024-01-10T17:59:09.694Z 19 380-399 88 88 88 168 168 168 F F F
97852024-01-10T17:59:09.694Z 20 400-419 87 87 87 163 163 163 F F F
97862024-01-10T17:59:09.694Z 21 420-439 84 90 90 156 172 172 F F F <---
97872024-01-10T17:59:09.694Z 22 440-459 89 89 89 169 169 169 F F F
97882024-01-10T17:59:09.694Z 23 460-479 77 77 77 142 142 142 F F F
97892024-01-10T17:59:09.694Z 24 480-499 85 90 90 159 173 173 F F F <---
97902024-01-10T17:59:09.695Z 25 500-519 89 90 90 169 173 173 F F F <---
97912024-01-10T17:59:09.695Z 26 520-539 88 90 90 168 172 172 F F F <---
97922024-01-10T17:59:09.695Z 27 540-559 82 82 82 151 151 151 F F F
97932024-01-10T17:59:09.695Z 28 560-579 85 85 85 159 159 159 F F F
97942024-01-10T17:59:09.695Z 29 580-599 86 86 86 162 162 162 F F F
97952024-01-10T17:59:09.695ZMax gen: 90, Max flush: 173
97962024-01-10T17:59:09.695ZError: Difference in extent metadata found!
97972024-01-10T17:59:09.695ZA Difference in extent metadata is expected here
97982024-01-10T17:59:09.695ZOn loop 45
97992024-01-10T17:59:09.695Z
98002024-01-10T17:59:09.695ZVerifying data now
98012024-01-10T17:59:15.825ZNo change after verify
98022024-01-10T17:59:15.825ZLoop: 45 Downstairs dump after verify (and repair):
98032024-01-10T17:59:15.847ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98042024-01-10T17:59:15.847Z 0 000-019 91 91 91 174 174 174 F F F
98052024-01-10T17:59:15.847Z 1 020-039 89 89 89 169 169 169 F F F
98062024-01-10T17:59:15.848Z 2 040-059 91 91 91 174 174 174 F F F
98072024-01-10T17:59:15.848Z 3 060-079 91 91 91 174 174 174 F F F
98082024-01-10T17:59:15.848Z 4 080-099 87 87 87 163 163 163 F F F
98092024-01-10T17:59:15.848Z 5 100-119 90 90 90 170 170 170 F F F
98102024-01-10T17:59:15.848Z 6 120-139 91 91 91 174 174 174 F F F
98112024-01-10T17:59:15.889Z 7 140-159 91 91 91 174 174 174 F F F
98122024-01-10T17:59:15.889Z 8 160-179 81 81 81 150 150 150 F F F
98132024-01-10T17:59:15.889Z 9 180-199 90 90 90 170 170 170 F F F
98142024-01-10T17:59:15.889Z 10 200-219 90 90 90 172 172 172 F F F
98152024-01-10T17:59:15.889Z 11 220-239 86 86 86 160 160 160 F F F
98162024-01-10T17:59:15.889Z 12 240-259 86 86 86 160 160 160 F F F
98172024-01-10T17:59:15.889Z 13 260-279 90 90 90 170 170 170 F F F
98182024-01-10T17:59:15.889Z 14 280-299 91 91 91 174 174 174 F F F
98192024-01-10T17:59:15.889Z 15 300-319 84 84 84 154 154 154 F F F
98202024-01-10T17:59:15.890Z 16 320-339 90 90 90 173 173 173 F F F
98212024-01-10T17:59:15.890Z 17 340-359 90 90 90 173 173 173 F F F
98222024-01-10T17:59:15.890Z 18 360-379 88 88 88 167 167 167 F F F
98232024-01-10T17:59:15.890Z 19 380-399 88 88 88 168 168 168 F F F
98242024-01-10T17:59:15.891Z 20 400-419 87 87 87 163 163 163 F F F
98252024-01-10T17:59:15.891Z 21 420-439 90 90 90 172 172 172 F F F
98262024-01-10T17:59:15.891Z 22 440-459 89 89 89 169 169 169 F F F
98272024-01-10T17:59:15.891Z 23 460-479 77 77 77 142 142 142 F F F
98282024-01-10T17:59:15.891Z 24 480-499 90 90 90 173 173 173 F F F
98292024-01-10T17:59:15.891Z 25 500-519 90 90 90 173 173 173 F F F
98302024-01-10T17:59:15.891Z 26 520-539 90 90 90 172 172 172 F F F
98312024-01-10T17:59:15.891Z 27 540-559 82 82 82 151 151 151 F F F
98322024-01-10T17:59:15.891Z 28 560-579 85 85 85 159 159 159 F F F
98332024-01-10T17:59:15.891Z 29 580-599 86 86 86 162 162 162 F F F
98342024-01-10T17:59:15.891ZMax gen: 91, Max flush: 174
98352024-01-10T17:59:15.891Z
98362024-01-10T17:59:15.891ZBegin loop 46
98372024-01-10T17:59:15.891ZDownstairs to restart: 0
98382024-01-10T17:59:15.898Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.883748667Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523}
98392024-01-10T17:59:15.898Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.884238275Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523}
98402024-01-10T17:59:15.898Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.884288096Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523}
98412024-01-10T17:59:15.898Z{"msg":"Crucible dba2b4c6-1483-4e40-9b0c-1d1a431347a7 has session id: 62f31779-47fc-4bf4-a3b5-d4bfde336dfd","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.884311123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98422024-01-10T17:59:15.898Z{"msg":"Upstairs opts: Upstairs UUID: dba2b4c6-1483-4e40-9b0c-1d1a431347a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.884345732Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98432024-01-10T17:59:15.898Z{"msg":"Crucible stats registered with UUID: dba2b4c6-1483-4e40-9b0c-1d1a431347a7","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.884369056Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98442024-01-10T17:59:15.899ZCrucible runtime is spawned
98452024-01-10T17:59:15.899Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.884719802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523}
98462024-01-10T17:59:15.899Z{"msg":"dba2b4c6-1483-4e40-9b0c-1d1a431347a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.884930993Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98472024-01-10T17:59:15.899Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.884996791Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"0","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98482024-01-10T17:59:15.899Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.885069421Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"0","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98492024-01-10T17:59:15.899Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.88511667Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"1","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98502024-01-10T17:59:15.899Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.885190188Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"1","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98512024-01-10T17:59:15.899Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.885231967Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"2","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98522024-01-10T17:59:15.899Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.885333342Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"2","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98532024-01-10T17:59:15.899Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.885866216Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98542024-01-10T17:59:15.899Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.885909713Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98552024-01-10T17:59:15.899Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.886013819Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98562024-01-10T17:59:15.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.886058695Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98572024-01-10T17:59:15.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.886118576Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98582024-01-10T17:59:15.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.88614766Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98592024-01-10T17:59:15.899Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.88687452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"1","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98602024-01-10T17:59:15.899Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.8870014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"1","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98612024-01-10T17:59:15.899Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.887084734Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"2","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98622024-01-10T17:59:15.899Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.887126625Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"2","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98632024-01-10T17:59:15.902Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.88717054Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"0","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98642024-01-10T17:59:15.902Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.887199554Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"0","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98652024-01-10T17:59:15.902Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.887945668Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"1","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98662024-01-10T17:59:15.902Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.888038844Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"2","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98672024-01-10T17:59:15.902Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.888463757Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"0","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98682024-01-10T17:59:15.902Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.888845604Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"2","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98692024-01-10T17:59:15.902Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.888917861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"2","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98702024-01-10T17:59:15.902Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.888949912Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98712024-01-10T17:59:15.902Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.888993641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"1","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98722024-01-10T17:59:15.902Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889018388Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"1","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98732024-01-10T17:59:15.903Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889064868Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98742024-01-10T17:59:15.903Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889310618Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"0","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98752024-01-10T17:59:15.903Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.88937794Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"0","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98762024-01-10T17:59:15.903Z{"msg":"[0]R flush_numbers[0..12]: [174, 169, 174, 174, 163, 170, 174, 174, 150, 170, 172, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889425285Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98772024-01-10T17:59:15.903Z{"msg":"[0]R generation[0..12]: [91, 89, 91, 91, 87, 90, 91, 91, 81, 90, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889480108Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98782024-01-10T17:59:15.903Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889526005Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98792024-01-10T17:59:15.905Z{"msg":"[1]R flush_numbers[0..12]: [174, 169, 174, 174, 163, 170, 174, 174, 150, 170, 172, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889583955Z","Wait for a query_work_queue command to finish before sending IO
98802024-01-10T17:59:15.905Zhostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98812024-01-10T17:59:15.905Z{"msg":"[1]R generation[0..12]: [91, 89, 91, 91, 87, 90, 91, 91, 81, 90, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889632612Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98822024-01-10T17:59:15.905Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889700277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98832024-01-10T17:59:15.907Z{"msg":"[2]R flush_numbers[0..12]: [174, 169, 174, 174, 163, 170, 174, 174, 150, 170, 172, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889778814Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98842024-01-10T17:59:15.907Z{"msg":"[2]R generation[0..12]: [91, 89, 91, 91, 87, 90, 91, 91, 81, 90, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889821359Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98852024-01-10T17:59:15.907Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889879578Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98862024-01-10T17:59:15.907Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889921487Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98872024-01-10T17:59:15.907Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.889978864Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98882024-01-10T17:59:15.907Z{"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.890019881Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98892024-01-10T17:59:15.907Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.890072218Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"":"downstairs","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98902024-01-10T17:59:15.908Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.890114094Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98912024-01-10T17:59:15.908Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
98922024-01-10T17:59:15.908Z,"time":"2024-01-10T17:59:15.890174447Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"0","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98932024-01-10T17:59:15.908Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.890222305Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"1","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98942024-01-10T17:59:15.908Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.890273861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"client":"2","":"downstairs","session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98952024-01-10T17:59:15.908Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.89031594Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98962024-01-10T17:59:15.908Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.890366614Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98972024-01-10T17:59:15.908Z{"msg":"dba2b4c6-1483-4e40-9b0c-1d1a431347a7 is now active with session: 62f31779-47fc-4bf4-a3b5-d4bfde336dfd","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.890407229Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98982024-01-10T17:59:15.908Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.890457461Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523,"session_id":"62f31779-47fc-4bf4-a3b5-d4bfde336dfd"}
98992024-01-10T17:59:15.908Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:15.890496118Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1523}
99002024-01-10T17:59:15.908ZLoading write count information from file "/tmp/test_repair_verify.data"
99012024-01-10T17:59:15.908ZRun Repair workload
99022024-01-10T17:59:15.908Z01/30 Write block 420 len 28672 data: 14 12 11 11 9 7 6
99032024-01-10T17:59:15.908Z02/30 Flush
99042024-01-10T17:59:15.908Z03/30 Write block 65 len 28672 data: 9 10 8 7 6 7 4
99052024-01-10T17:59:15.908Z04/30 Read block 214 len 28672
99062024-01-10T17:59:15.912Z05/30 Read block 309 len 16384
99072024-01-10T17:59:15.912Z06/30 Read block 313 len 8192
99082024-01-10T17:59:15.912Z07/30 Write block 117 len 24576 data: 6 5 7 8 7 7
99092024-01-10T17:59:15.913Z08/30 Read block 68 len 24576
99102024-01-10T17:59:15.913Z09/30 Read block 157 len 32768
99112024-01-10T17:59:15.913Z10/30 Read block 346 len 12288
99122024-01-10T17:59:15.913Z11/30 Read block 278 len 24576
99132024-01-10T17:59:15.913Z12/30 Read block 31 len 16384
99142024-01-10T17:59:15.916Z13/30 Write block 453 len 12288 data: 10 9 7
99152024-01-10T17:59:15.917Z14/30 Read block 586 len 40960
99162024-01-10T17:59:15.919Z15/30 Read block 388 len 36864
99172024-01-10T17:59:15.923Z16/30 Read block 59 len 12288
99182024-01-10T17:59:15.923Z17/30 Read block 439 len 28672
99192024-01-10T17:59:15.926Z18/30 Read block 5 len 36864
99202024-01-10T17:59:15.929Z19/30 Read block 289 len 12288
99212024-01-10T17:59:15.933Z20/30 Write block 50 len 40960 data: 11 8 8 6 9 9 8 7 9 9
99222024-01-10T17:59:15.933Z21/30 Read block 416 len 20480
99232024-01-10T17:59:15.937Z22/30 Write block 362 len 28672 data: 6 8 7 5 5 5 6
99242024-01-10T17:59:15.937Z23/30 Flush
99252024-01-10T17:59:15.940Z24/30 Write block 408 len 40960 data: 9 9 11 12 9 9 9 7 7 8
99262024-01-10T17:59:15.940Z25/30 Read block 304 len 4096
99272024-01-10T17:59:15.943Z26/30 Read block 234 len 12288
99282024-01-10T17:59:15.947Z27/30 Read block 336 len 36864
99292024-01-10T17:59:15.947Z28/30 Read block 142 len 20480
99302024-01-10T17:59:15.950Z29/30 Write block 134 len 32768 data: 7 8 6 7 7 8 9 9
99312024-01-10T17:59:15.953Z30/30 Read block 337 len 24576
99322024-01-10T17:59:15.953Z----------------------------------------------------------------
99332024-01-10T17:59:15.953Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:28
99342024-01-10T17:59:15.953ZStates: Active Active Active
99352024-01-10T17:59:15.954ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99362024-01-10T17:59:15.954Z 3 Acked 1002 Write 7 Sent Done Done false
99372024-01-10T17:59:15.954Z 4 Acked 1003 Read 7 Sent Done Done false
99382024-01-10T17:59:15.954Z 5 Acked 1004 Read 4 Sent Done Done false
99392024-01-10T17:59:15.954Z 6 Acked 1005 Read 2 Sent Done Done false
99402024-01-10T17:59:15.954Z 7 Acked 1006 Write 6 Sent Done Done false
99412024-01-10T17:59:15.954Z 8 Acked 1007 Read 6 Sent Done Done false
99422024-01-10T17:59:15.954Z 9 Acked 1008 Read 8 Sent Done Done false
99432024-01-10T17:59:15.954Z 10 Acked 1009 Read 3 Sent Done Done false
99442024-01-10T17:59:15.954Z 11 Acked 1010 Read 6 Sent Done Done false
99452024-01-10T17:59:15.954Z 12 Acked 1011 Read 4 Sent Done Done false
99462024-01-10T17:59:15.954Z 13 Acked 1012 Write 3 Sent Done Done false
99472024-01-10T17:59:15.954Z 14 Acked 1013 Read 10 Sent Done Done false
99482024-01-10T17:59:15.954Z 15 Acked 1014 Read 9 Sent Done Done false
99492024-01-10T17:59:15.954Z 16 Acked 1015 Read 3 Sent Done Done false
99502024-01-10T17:59:15.954Z 17 Acked 1016 Read 7 Sent Done Done false
99512024-01-10T17:59:15.954Z 18 Acked 1017 Read 9 Sent Done Done false
99522024-01-10T17:59:15.954Z 19 Acked 1018 Read 3 Sent Done Done false
99532024-01-10T17:59:15.955Z 20 Acked 1019 Write 10 Sent Done Done false
99542024-01-10T17:59:15.955Z 21 Acked 1020 Read 5 Sent Done Done false
99552024-01-10T17:59:15.955Z 22 Acked 1021 Write 7 Sent Done Done false
99562024-01-10T17:59:15.955Z 23 Acked 1022 Flush 0 Sent Done Done false
99572024-01-10T17:59:15.955Z 24 Acked 1023 Write 10 Sent Done Done false
99582024-01-10T17:59:15.955Z 25 Acked 1024 Read 1 Sent Done Done false
99592024-01-10T17:59:15.959Z 26 Acked 1025 Read 3 Sent Done Done false
99602024-01-10T17:59:15.959Z 27 Acked 1026 Read 9 Sent Done Done false
99612024-01-10T17:59:15.959Z 28 Acked 1027 Read 5 Sent Done Done false
99622024-01-10T17:59:15.959Z 29 Acked 1028 Write 8 Sent Done Done false
99632024-01-10T17:59:15.959Z 30 Acked 1029 Read 6 Sent Done Sent false
99642024-01-10T17:59:15.959Z STATES DS:0 DS:1 DS:2 TOTAL
99652024-01-10T17:59:15.959Z New 0 0 0 0
99662024-01-10T17:59:15.959Z Sent 28 0 1 29
99672024-01-10T17:59:15.959Z Done 0 28 27 55
99682024-01-10T17:59:15.959Z Skipped 0 0 0 0
99692024-01-10T17:59:15.959Z Error 0 0 0 0
99702024-01-10T17:59:15.959ZLast Flush: 1001 1022 1022
99712024-01-10T17:59:15.959ZDownstairs last five completed: 1001 1000
99722024-01-10T17:59:15.959ZUpstairs last five completed: 30 29 28 27 26
99732024-01-10T17:59:15.959ZWrote out file "/tmp/test_repair_verify.data"
99742024-01-10T17:59:15.962Z
99752024-01-10T17:59:15.965ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
99762024-01-10T17:59:15.988ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99772024-01-10T17:59:15.988Z 0 000-019 91 91 91 174 174 174 F F F
99782024-01-10T17:59:15.988Z 1 020-039 89 89 89 169 169 169 F F F
99792024-01-10T17:59:15.988Z 2 040-059 91 92 92 174 176 176 F F F <---
99802024-01-10T17:59:15.988Z 3 060-079 91 92 92 174 176 176 F F F <---
99812024-01-10T17:59:15.989Z 4 080-099 87 87 87 163 163 163 F F F
99822024-01-10T17:59:15.989Z 5 100-119 90 92 92 170 176 176 F F F <---
99832024-01-10T17:59:15.989Z 6 120-139 91 92 92 174 176 176 F T T <---
99842024-01-10T17:59:15.994Z 7 140-159 91 91 91 174 174 174 F T T <---
99852024-01-10T17:59:15.994Z 8 160-179 81 81 81 150 150 150 F F F
99862024-01-10T17:59:15.994Z 9 180-199 90 90 90 170 170 170 F F F
99872024-01-10T17:59:15.994Z 10 200-219 90 90 90 172 172 172 F F F
99882024-01-10T17:59:15.994Z 11 220-239 86 86 86 160 160 160 F F F
99892024-01-10T17:59:15.994Z 12 240-259 86 86 86 160 160 160 F F F
99902024-01-10T17:59:15.994Z 13 260-279 90 90 90 170 170 170 F F F
99912024-01-10T17:59:15.994Z 14 280-299 91 91 91 174 174 174 F F F
99922024-01-10T17:59:15.994Z 15 300-319 84 84 84 154 154 154 F F F
99932024-01-10T17:59:15.994Z 16 320-339 90 90 90 173 173 173 F F F
99942024-01-10T17:59:15.994Z 17 340-359 90 90 90 173 173 173 F F F
99952024-01-10T17:59:15.995Z 18 360-379 88 92 92 167 176 176 F F F <---
99962024-01-10T17:59:15.995Z 19 380-399 88 88 88 168 168 168 F F F
99972024-01-10T17:59:15.995Z 20 400-419 87 87 87 163 163 163 F T T <---
99982024-01-10T17:59:15.995Z 21 420-439 92 92 92 175 175 175 F F F
99992024-01-10T17:59:15.995Z 22 440-459 89 92 92 169 176 176 F F F <---
100002024-01-10T17:59:15.995Z 23 460-479 77 77 77 142 142 142 F F F
100012024-01-10T17:59:15.995Z 24 480-499 90 90 90 173 173 173 F F F
100022024-01-10T17:59:15.995Z 25 500-519 90 90 90 173 173 173 F F F
100032024-01-10T17:59:15.995Z 26 520-539 90 90 90 172 172 172 F F F
100042024-01-10T17:59:15.995Z 27 540-559 82 82 82 151 151 151 F F F
100052024-01-10T17:59:15.996Z 28 560-579 85 85 85 159 159 159 F F F
100062024-01-10T17:59:15.996Z 29 580-599 86 86 86 162 162 162 F F F
100072024-01-10T17:59:15.996ZMax gen: 92, Max flush: 176
100082024-01-10T17:59:15.996ZError: Difference in extent metadata found!
100092024-01-10T17:59:15.996ZA Difference in extent metadata is expected here
100102024-01-10T17:59:15.996ZOn loop 46
100112024-01-10T17:59:15.996Z
100122024-01-10T17:59:16.005ZVerifying data now
100132024-01-10T17:59:20.881ZNo change after verify
100142024-01-10T17:59:20.881ZLoop: 46 Downstairs dump after verify (and repair):
100152024-01-10T17:59:20.903ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100162024-01-10T17:59:20.903Z 0 000-019 91 91 91 174 174 174 F F F
100172024-01-10T17:59:20.903Z 1 020-039 89 89 89 169 169 169 F F F
100182024-01-10T17:59:20.903Z 2 040-059 92 92 92 176 176 176 F F F
100192024-01-10T17:59:20.903Z 3 060-079 92 92 92 176 176 176 F F F
100202024-01-10T17:59:20.903Z 4 080-099 87 87 87 163 163 163 F F F
100212024-01-10T17:59:20.904Z 5 100-119 92 92 92 176 176 176 F F F
100222024-01-10T17:59:20.904Z 6 120-139 93 93 93 177 177 177 F F F
100232024-01-10T17:59:20.916Z 7 140-159 93 93 93 177 177 177 F F F
100242024-01-10T17:59:20.917Z 8 160-179 81 81 81 150 150 150 F F F
100252024-01-10T17:59:20.917Z 9 180-199 90 90 90 170 170 170 F F F
100262024-01-10T17:59:20.917Z 10 200-219 90 90 90 172 172 172 F F F
100272024-01-10T17:59:20.917Z 11 220-239 86 86 86 160 160 160 F F F
100282024-01-10T17:59:20.917Z 12 240-259 86 86 86 160 160 160 F F F
100292024-01-10T17:59:20.918Z 13 260-279 90 90 90 170 170 170 F F F
100302024-01-10T17:59:20.918Z 14 280-299 91 91 91 174 174 174 F F F
100312024-01-10T17:59:20.918Z 15 300-319 84 84 84 154 154 154 F F F
100322024-01-10T17:59:20.918Z 16 320-339 90 90 90 173 173 173 F F F
100332024-01-10T17:59:20.918Z 17 340-359 90 90 90 173 173 173 F F F
100342024-01-10T17:59:20.918Z 18 360-379 92 92 92 176 176 176 F F F
100352024-01-10T17:59:20.918Z 19 380-399 88 88 88 168 168 168 F F F
100362024-01-10T17:59:20.919Z 20 400-419 93 93 93 177 177 177 F F F
100372024-01-10T17:59:20.919Z 21 420-439 92 92 92 175 175 175 F F F
100382024-01-10T17:59:20.919Z 22 440-459 92 92 92 176 176 176 F F F
100392024-01-10T17:59:20.919Z 23 460-479 77 77 77 142 142 142 F F F
100402024-01-10T17:59:20.919Z 24 480-499 90 90 90 173 173 173 F F F
100412024-01-10T17:59:20.919Z 25 500-519 90 90 90 173 173 173 F F F
100422024-01-10T17:59:20.919Z 26 520-539 90 90 90 172 172 172 F F F
100432024-01-10T17:59:20.919Z 27 540-559 82 82 82 151 151 151 F F F
100442024-01-10T17:59:20.920Z 28 560-579 85 85 85 159 159 159 F F F
100452024-01-10T17:59:20.920Z 29 580-599 86 86 86 162 162 162 F F F
100462024-01-10T17:59:20.920ZMax gen: 93, Max flush: 177
100472024-01-10T17:59:20.920Z
100482024-01-10T17:59:20.920ZBegin loop 47
100492024-01-10T17:59:20.920ZDownstairs to restart: 2
100502024-01-10T17:59:20.941Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.939789908Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531}
100512024-01-10T17:59:20.949Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.940331924Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531}
100522024-01-10T17:59:20.949Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.940414104Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531}
100532024-01-10T17:59:20.949Z{"msg":"Crucible 64a5e497-33f3-4a15-9e0c-bfe57aa63439 has session id: 0532cc50-7d65-4187-81ec-d20177688a21","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.94048126Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100542024-01-10T17:59:20.949Z{"msg":"Upstairs opts: Upstairs UUID: 64a5e497-33f3-4a15-9e0c-bfe57aa63439, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.940541298Z"Crucible runtime is spawned
100552024-01-10T17:59:20.949Z,"hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100562024-01-10T17:59:20.949Z{"msg":"Crucible stats registered with UUID: 64a5e497-33f3-4a15-9e0c-bfe57aa63439","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.940642833Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100572024-01-10T17:59:20.950Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.94071301Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531}
100582024-01-10T17:59:20.950Z{"msg":"64a5e497-33f3-4a15-9e0c-bfe57aa63439 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.940818987Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100592024-01-10T17:59:20.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.940882548Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"0","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100602024-01-10T17:59:20.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.940999484Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"0","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100612024-01-10T17:59:20.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.941037532Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"1","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100622024-01-10T17:59:20.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.941072603Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"1","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100632024-01-10T17:59:20.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.941236485Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"2","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100642024-01-10T17:59:20.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.941461762Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"2","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100652024-01-10T17:59:20.950Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.941546245Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100662024-01-10T17:59:20.951Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.941594988Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100672024-01-10T17:59:20.951Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.941678368Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100682024-01-10T17:59:20.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.941722413Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100692024-01-10T17:59:20.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.941819408Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100702024-01-10T17:59:20.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.941863361Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100712024-01-10T17:59:20.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.942649412Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"1","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100722024-01-10T17:59:20.951Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.94281583Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"1","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100732024-01-10T17:59:20.952Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.942895549Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"0","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100742024-01-10T17:59:20.952Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.942987183Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"0","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100752024-01-10T17:59:20.952Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.943702145Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"1","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100762024-01-10T17:59:20.952Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.943784695Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"0","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100772024-01-10T17:59:20.952Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.943840556Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"2","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100782024-01-10T17:59:20.952Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.943868029Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"2","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100792024-01-10T17:59:20.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.9447649Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"0","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100802024-01-10T17:59:20.952Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.944827713Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"0","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100812024-01-10T17:59:20.952Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.944854357Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100822024-01-10T17:59:20.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.94487492Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"1","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100832024-01-10T17:59:20.952Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.944915826Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"1","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100842024-01-10T17:59:20.952Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.944939169Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100852024-01-10T17:59:20.952Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.944960539Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"2","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100862024-01-10T17:59:20.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945620958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"2","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100872024-01-10T17:59:20.952Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945675802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"2","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100882024-01-10T17:59:20.952Z{"msg":"[0]R flush_numbers[0..12]: [174, 169, 176, 176, 163, 176, 177, 177, 150, 170, 172, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945701721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100892024-01-10T17:59:20.953Z{"msg":"[0]R generation[0..12]: [91, 89, 92, 92, 87, 92, 93, 93, 81, 90, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945724691Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100902024-01-10T17:59:20.953Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945766307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100912024-01-10T17:59:20.953Z{"msg":"[1]R flush_numbers[0..12]: [174, 169, 176, 176, 163, 176, 177, 177, 150, 170, 172, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945791118Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100922024-01-10T17:59:20.953Z{"msg":"[1]R generation[0..12]: [91, 89, 92, 92, 87, 92, 93, 93, 81, 90, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945813774Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100932024-01-10T17:59:20.953Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945854945Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100942024-01-10T17:59:20.953Z{"msg":"[2]R flush_numbers[0..12]: [174, 169, 176, 176, 163, 176, 177, 177, 150, 170, 172, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945879529Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100952024-01-10T17:59:20.953Z{"msg":"[2]R generation[0..12]: [91, 89, 92, 92, 87, 92, 93, 93, 81, 90, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945930334Z","hostname":"Wait for a query_work_queue command to finish before sending IO
100962024-01-10T17:59:20.953Zip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100972024-01-10T17:59:20.953Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945961196Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100982024-01-10T17:59:20.953Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.945984426Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
100992024-01-10T17:59:20.954Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946006313Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101002024-01-10T17:59:20.954Z{"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946027179Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101012024-01-10T17:59:20.954Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946048436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"":"downstairs","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101022024-01-10T17:59:20.954Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946069705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101032024-01-10T17:59:20.954Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946108013Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"0","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101042024-01-10T17:59:20.954Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946131089Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"1","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101052024-01-10T17:59:20.955Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946152637Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"client":"2","":"downstairs","session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101062024-01-10T17:59:20.955Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946191837Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101072024-01-10T17:59:20.955Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946213032Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101082024-01-10T17:59:20.955Z{"msg":"64a5e497-33f3-4a15-9e0c-bfe57aa63439 is now active with session: 0532cc50-7d65-4187-81ec-d20177688a21","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.94623424Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101092024-01-10T17:59:20.955Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946271182Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531,"session_id":"0532cc50-7d65-4187-81ec-d20177688a21"}
101102024-01-10T17:59:20.955Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:20.946292786Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1531}
101112024-01-10T17:59:20.955ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101122024-01-10T17:59:20.955ZLoading write count information from file "/tmp/test_repair_verify.data"
101132024-01-10T17:59:20.955ZRun Repair workload
101142024-01-10T17:59:20.955Z01/30 Flush
101152024-01-10T17:59:20.955Z02/30 Write block 548 len 40960 data: 11 12 11 11 14 11 10 8 6 6
101162024-01-10T17:59:20.958Z03/30 Write block 285 len 8192 data: 9 7
101172024-01-10T17:59:20.959Z04/30 Flush
101182024-01-10T17:59:20.959Z05/30 Write block 187 len 32768 data: 10 12 12 11 11 11 12 13
101192024-01-10T17:59:20.959Z06/30 Read block 490 len 4096
101202024-01-10T17:59:20.959Z07/30 Read block 237 len 32768
101212024-01-10T17:59:20.959Z08/30 Write block 376 len 28672 data: 6 5 4 5 6 7 7
101222024-01-10T17:59:20.959Z09/30 Read block 251 len 40960
101232024-01-10T17:59:20.962Z10/30 Read block 168 len 32768
101242024-01-10T17:59:20.965Z11/30 Read block 552 len 40960
101252024-01-10T17:59:20.971Z12/30 Read block 310 len 8192
101262024-01-10T17:59:20.974Z13/30 Read block 53 len 16384
101272024-01-10T17:59:20.974Z14/30 Write block 311 len 28672 data: 5 2 3 4 7 7 5
101282024-01-10T17:59:20.974Z15/30 Flush
101292024-01-10T17:59:20.977Z16/30 Write block 321 len 20480 data: 7 7 8 9 9
101302024-01-10T17:59:20.977Z17/30 Read block 480 len 8192
101312024-01-10T17:59:20.981Z18/30 Write block 333 len 40960 data: 3 4 6 7 8 8 9 8 8 11
101322024-01-10T17:59:20.981Z19/30 Write block 514 len 12288 data: 14 13 11
101332024-01-10T17:59:20.984Z20/30 Write block 381 len 24576 data: 8 8 5 5 5 5
101342024-01-10T17:59:20.984Z21/30 Flush
101352024-01-10T17:59:20.987Z22/30 Write block 546 len 24576 data: 7 12 12 13 12 12
101362024-01-10T17:59:20.987Z23/30 Write block 365 len 12288 data: 6 6 6
101372024-01-10T17:59:20.987Z24/30 Read block 211 len 4096
101382024-01-10T17:59:20.990Z25/30 Read block 85 len 24576
101392024-01-10T17:59:20.993Z26/30 Write block 132 len 8192 data: 7 7
101402024-01-10T17:59:20.993Z27/30 Read block 538 len 12288
101412024-01-10T17:59:20.994Z28/30 Write block 390 len 4096 data: 6
101422024-01-10T17:59:20.997Z29/30 Read block 483 len 16384
101432024-01-10T17:59:20.997Z30/30 Read block 156 len 40960
101442024-01-10T17:59:21.001Z----------------------------------------------------------------
101452024-01-10T17:59:21.001Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:29
101462024-01-10T17:59:21.001ZStates: Active Active Active
101472024-01-10T17:59:21.002ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
101482024-01-10T17:59:21.002Z 2 Acked 1001 Write 10 Done Done Sent false
101492024-01-10T17:59:21.002Z 3 Acked 1002 Write 2 Done Done Sent false
101502024-01-10T17:59:21.002Z 4 Acked 1003 Flush 0 Done Done Sent false
101512024-01-10T17:59:21.002Z 5 Acked 1004 Write 8 Done Done Sent false
101522024-01-10T17:59:21.002Z 6 Acked 1005 Read 1 Done Done Sent false
101532024-01-10T17:59:21.002Z 7 Acked 1006 Read 8 Done Done Sent false
101542024-01-10T17:59:21.002Z 8 Acked 1007 Write 7 Done Done Sent false
101552024-01-10T17:59:21.002Z 9 Acked 1008 Read 10 Done Done Sent false
101562024-01-10T17:59:21.002Z 10 Acked 1009 Read 8 Done Done Sent false
101572024-01-10T17:59:21.002Z 11 Acked 1010 Read 10 Done Done Sent false
101582024-01-10T17:59:21.002Z 12 Acked 1011 Read 2 Done Done Sent false
101592024-01-10T17:59:21.002Z 13 Acked 1012 Read 4 Done Done Sent false
101602024-01-10T17:59:21.002Z 14 Acked 1013 Write 7 Done Done Sent false
101612024-01-10T17:59:21.002Z 15 Acked 1014 Flush 0 Done Done Sent false
101622024-01-10T17:59:21.002Z 16 Acked 1015 Write 5 Done Done Sent false
101632024-01-10T17:59:21.002Z 17 Acked 1016 Read 2 Done Done Sent false
101642024-01-10T17:59:21.002Z 18 Acked 1017 Write 10 Done Done Sent false
101652024-01-10T17:59:21.002Z 19 Acked 1018 Write 3 Done Done Sent false
101662024-01-10T17:59:21.002Z 20 Acked 1019 Write 6 Done Done Sent false
101672024-01-10T17:59:21.002Z 21 Acked 1020 Flush 0 Done Done Sent false
101682024-01-10T17:59:21.002Z 22 Acked 1021 Write 6 Done Done Sent false
101692024-01-10T17:59:21.002Z 23 Acked 1022 Write 3 Done Done Sent false
101702024-01-10T17:59:21.003Z 24 Acked 1023 Read 1 Done Done Sent false
101712024-01-10T17:59:21.003Z 25 Acked 1024 Read 6 Done Done Sent false
101722024-01-10T17:59:21.003Z 26 Acked 1025 Write 2 Done Done Sent false
101732024-01-10T17:59:21.003Z 27 Acked 1026 Read 3 Done Done Sent false
101742024-01-10T17:59:21.003Z 28 Acked 1027 Write 1 Done Done Sent false
101752024-01-10T17:59:21.003Z 29 Acked 1028 Read 4 Done Done Sent false
101762024-01-10T17:59:21.003Z 30 Acked 1029 Read 10 Done Sent Sent false
101772024-01-10T17:59:21.003Z STATES DS:0 DS:1 DS:2 TOTAL
101782024-01-10T17:59:21.003Z New 0 0 0 0
101792024-01-10T17:59:21.003Z Sent 0 1 29 30
101802024-01-10T17:59:21.003Z Done 29 28 0 57
101812024-01-10T17:59:21.003Z Skipped 0 0 0 0
101822024-01-10T17:59:21.003Z Error 0 0 0 0
101832024-01-10T17:59:21.003ZLast Flush: 1020 1020 1000
101842024-01-10T17:59:21.003ZDownstairs last five completed: 1000
101852024-01-10T17:59:21.003ZUpstairs last five completed: 30 29 28 27 26
101862024-01-10T17:59:21.003ZWrote out file "/tmp/test_repair_verify.data"
101872024-01-10T17:59:21.007Z
101882024-01-10T17:59:21.012ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
101892024-01-10T17:59:21.058ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101902024-01-10T17:59:21.058Z 0 000-019 91 91 91 174 174 174 F F F
101912024-01-10T17:59:21.058Z 1 020-039 89 89 89 169 169 169 F F F
101922024-01-10T17:59:21.058Z 2 040-059 92 92 92 176 176 176 F F F
101932024-01-10T17:59:21.058Z 3 060-079 92 92 92 176 176 176 F F F
101942024-01-10T17:59:21.062Z 4 080-099 87 87 87 163 163 163 F F F
101952024-01-10T17:59:21.062Z 5 100-119 92 92 92 176 176 176 F F F
101962024-01-10T17:59:21.062Z 6 120-139 93 93 93 177 177 177 T T F <---
101972024-01-10T17:59:21.062Z 7 140-159 93 93 93 177 177 177 F F F
101982024-01-10T17:59:21.062Z 8 160-179 81 81 81 150 150 150 F F F
101992024-01-10T17:59:21.062Z 9 180-199 94 94 90 180 180 170 F F F <---
102002024-01-10T17:59:21.062Z 10 200-219 90 90 90 172 172 172 F F F
102012024-01-10T17:59:21.062Z 11 220-239 86 86 86 160 160 160 F F F
102022024-01-10T17:59:21.062Z 12 240-259 86 86 86 160 160 160 F F F
102032024-01-10T17:59:21.062Z 13 260-279 90 90 90 170 170 170 F F F
102042024-01-10T17:59:21.062Z 14 280-299 94 94 91 179 179 174 F F F <---
102052024-01-10T17:59:21.062Z 15 300-319 94 94 84 180 180 154 F F F <---
102062024-01-10T17:59:21.062Z 16 320-339 94 94 90 181 181 173 F F F <---
102072024-01-10T17:59:21.062Z 17 340-359 94 94 90 181 181 173 F F F <---
102082024-01-10T17:59:21.062Z 18 360-379 94 94 92 180 180 176 T T F <---
102092024-01-10T17:59:21.062Z 19 380-399 94 94 88 181 181 168 T T F <---
102102024-01-10T17:59:21.062Z 20 400-419 93 93 93 177 177 177 F F F
102112024-01-10T17:59:21.062Z 21 420-439 92 92 92 175 175 175 F F F
102122024-01-10T17:59:21.062Z 22 440-459 92 92 92 176 176 176 F F F
102132024-01-10T17:59:21.063Z 23 460-479 77 77 77 142 142 142 F F F
102142024-01-10T17:59:21.063Z 24 480-499 90 90 90 173 173 173 F F F
102152024-01-10T17:59:21.063Z 25 500-519 94 94 90 181 181 173 F F F <---
102162024-01-10T17:59:21.063Z 26 520-539 90 90 90 172 172 172 F F F
102172024-01-10T17:59:21.063Z 27 540-559 94 94 82 179 179 151 T T F <---
102182024-01-10T17:59:21.063Z 28 560-579 85 85 85 159 159 159 F F F
102192024-01-10T17:59:21.063Z 29 580-599 86 86 86 162 162 162 F F F
102202024-01-10T17:59:21.063ZMax gen: 94, Max flush: 181
102212024-01-10T17:59:21.063ZError: Difference in extent metadata found!
102222024-01-10T17:59:21.063ZA Difference in extent metadata is expected here
102232024-01-10T17:59:21.063ZOn loop 47
102242024-01-10T17:59:21.063Z
102252024-01-10T17:59:21.063ZVerifying data now
102262024-01-10T17:59:22.091ZNo change after verify
102272024-01-10T17:59:22.091ZLoop: 47 Downstairs dump after verify (and repair):
102282024-01-10T17:59:22.107ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102292024-01-10T17:59:22.107Z 0 000-019 91 91 91 174 174 174 F F F
102302024-01-10T17:59:22.107Z 1 020-039 89 89 89 169 169 169 F F F
102312024-01-10T17:59:22.107Z 2 040-059 92 92 92 176 176 176 F F F
102322024-01-10T17:59:22.107Z 3 060-079 92 92 92 176 176 176 F F F
102332024-01-10T17:59:22.107Z 4 080-099 87 87 87 163 163 163 F F F
102342024-01-10T17:59:22.111Z 5 100-119 92 92 92 176 176 176 F F F
102352024-01-10T17:59:22.111Z 6 120-139 95 95 95 182 182 182 F F F
102362024-01-10T17:59:22.111Z 7 140-159 93 93 93 177 177 177 F F F
102372024-01-10T17:59:22.111Z 8 160-179 81 81 81 150 150 150 F F F
102382024-01-10T17:59:22.111Z 9 180-199 94 94 94 180 180 180 F F F
102392024-01-10T17:59:22.111Z 10 200-219 90 90 90 172 172 172 F F F
102402024-01-10T17:59:22.111Z 11 220-239 86 86 86 160 160 160 F F F
102412024-01-10T17:59:22.111Z 12 240-259 86 86 86 160 160 160 F F F
102422024-01-10T17:59:22.111Z 13 260-279 90 90 90 170 170 170 F F F
102432024-01-10T17:59:22.111Z 14 280-299 94 94 94 179 179 179 F F F
102442024-01-10T17:59:22.111Z 15 300-319 94 94 94 180 180 180 F F F
102452024-01-10T17:59:22.111Z 16 320-339 94 94 94 181 181 181 F F F
102462024-01-10T17:59:22.111Z 17 340-359 94 94 94 181 181 181 F F F
102472024-01-10T17:59:22.111Z 18 360-379 95 95 95 182 182 182 F F F
102482024-01-10T17:59:22.111Z 19 380-399 95 95 95 182 182 182 F F F
102492024-01-10T17:59:22.111Z 20 400-419 93 93 93 177 177 177 F F F
102502024-01-10T17:59:22.111Z 21 420-439 92 92 92 175 175 175 F F F
102512024-01-10T17:59:22.111Z 22 440-459 92 92 92 176 176 176 F F F
102522024-01-10T17:59:22.113Z 23 460-479 77 77 77 142 142 142 F F F
102532024-01-10T17:59:22.113Z 24 480-499 90 90 90 173 173 173 F F F
102542024-01-10T17:59:22.113Z 25 500-519 94 94 94 181 181 181 F F F
102552024-01-10T17:59:22.113Z 26 520-539 90 90 90 172 172 172 F F F
102562024-01-10T17:59:22.113Z 27 540-559 95 95 95 182 182 182 F F F
102572024-01-10T17:59:22.113Z 28 560-579 85 85 85 159 159 159 F F F
102582024-01-10T17:59:22.113Z 29 580-599 86 86 86 162 162 162 F F F
102592024-01-10T17:59:22.113ZMax gen: 95, Max flush: 182
102602024-01-10T17:59:22.116Z
102612024-01-10T17:59:22.116ZBegin loop 48
102622024-01-10T17:59:22.116ZDownstairs to restart: 2
102632024-01-10T17:59:22.141Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.144399994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539}
102642024-01-10T17:59:22.144Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.144900018Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539}
102652024-01-10T17:59:22.144Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.144980459Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539}
102662024-01-10T17:59:22.144Z{"msg":"Crucible 6700a4c3-beb7-454c-9763-f6c597fddcbe has session id: 77a7b27d-3b88-4f50-b0b5-0cc565a91abf","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.145019575Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102672024-01-10T17:59:22.144Z{"msg":"Upstairs opts: Upstairs UUID: 6700a4c3-beb7-454c-9763-f6c597fddcbe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.145072769Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102682024-01-10T17:59:22.149Z{"msg":"Crucible stats registered with UUID: 6700a4c3-beb7-454c-9763-f6c597fddcbe","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.145148546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102692024-01-10T17:59:22.149ZCrucible runtime is spawned
102702024-01-10T17:59:22.149Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.145381403Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539}
102712024-01-10T17:59:22.149Z{"msg":"6700a4c3-beb7-454c-9763-f6c597fddcbe active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.145579291Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102722024-01-10T17:59:22.150Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.145644397Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"0","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102732024-01-10T17:59:22.150Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.145688933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"0","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102742024-01-10T17:59:22.150Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.145792475Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"1","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102752024-01-10T17:59:22.150Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.145936829Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"1","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102762024-01-10T17:59:22.150Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.146029297Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"2","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102772024-01-10T17:59:22.150Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.146102895Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"2","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102782024-01-10T17:59:22.151Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.146193183Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102792024-01-10T17:59:22.151Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.146235996Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102802024-01-10T17:59:22.151Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.14638658Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102812024-01-10T17:59:22.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.146423063Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102822024-01-10T17:59:22.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.146457628Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102832024-01-10T17:59:22.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.146486414Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102842024-01-10T17:59:22.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.147515969Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"0","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102852024-01-10T17:59:22.151Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.147600239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"0","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102862024-01-10T17:59:22.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.147699547Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"1","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102872024-01-10T17:59:22.151Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.147747583Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"1","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102882024-01-10T17:59:22.156Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.148333227Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"2","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102892024-01-10T17:59:22.156Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.148419232Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"2","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102902024-01-10T17:59:22.156Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.148446827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"0","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102912024-01-10T17:59:22.156Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.148664676Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"1","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102922024-01-10T17:59:22.156Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.149561617Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"1","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102932024-01-10T17:59:22.156Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.149637938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"1","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102942024-01-10T17:59:22.156Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.149684202Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102952024-01-10T17:59:22.156Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.14981893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"2","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102962024-01-10T17:59:22.156Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.149918316Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"0","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102972024-01-10T17:59:22.156Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.149945005Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"0","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102982024-01-10T17:59:22.156Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.150019518Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
102992024-01-10T17:59:22.157Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.150673452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"2","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103002024-01-10T17:59:22.157Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.150753317Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"2","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103012024-01-10T17:59:22.157Z{"msg":"[0]R flush_numbers[0..12]: [174, 169, 176, 176, 163, 176, 182, 177, 150, 180, 172, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.150784122Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103022024-01-10T17:59:22.157Z{"msg":"[0]R generation[0..12]: [91, 89, 92, 92, 87, 92, 95, 93, 81, 94, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.15080764Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103032024-01-10T17:59:22.157Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.150832004Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103042024-01-10T17:59:22.157Z{"msg":"[1]R flush_numbers[0..12]: [174, 169, 176, 176, 163, 176, 182, 177, 150, 180, 172, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.150855789Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103052024-01-10T17:59:22.157Z{"msg":"[1]R generation[0..12]: [91, 89, 92, 92, 87, 92, 95, 93, 81, 94, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.150878593Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103062024-01-10T17:59:22.157Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.150924565Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103072024-01-10T17:59:22.158Z{"msg":"[2]R flush_numbers[0..12]: [174, 169, 176, 176, 163, 176, 182, 177, 150, 180, 172, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.15095054Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103082024-01-10T17:59:22.159Z{"msg":"[2]R generation[0..12]: [91, 89, 92, 92, 87, 92, 95, 93, 81, 94, 90, 86]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
103092024-01-10T17:59:22.159Z,"time":"2024-01-10T17:59:22.151007555Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103102024-01-10T17:59:22.159Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151052726Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103112024-01-10T17:59:22.159Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151113231Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103122024-01-10T17:59:22.159Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151155463Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103132024-01-10T17:59:22.159Z{"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151222425Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103142024-01-10T17:59:22.159Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151253897Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"":"downstairs","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103152024-01-10T17:59:22.159Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151318267Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103162024-01-10T17:59:22.159Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151356967Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"0","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103172024-01-10T17:59:22.160Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151398942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"1","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103182024-01-10T17:59:22.160Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151442863Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"client":"2","":"downstairs","session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103192024-01-10T17:59:22.160Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151480465Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103202024-01-10T17:59:22.160Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151536701Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103212024-01-10T17:59:22.160Z{"msg":"6700a4c3-beb7-454c-9763-f6c597fddcbe is now active with session: 77a7b27d-3b88-4f50-b0b5-0cc565a91abf","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151561771Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103222024-01-10T17:59:22.160Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151599985Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539,"session_id":"77a7b27d-3b88-4f50-b0b5-0cc565a91abf"}
103232024-01-10T17:59:22.160Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:22.151621836Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1539}
103242024-01-10T17:59:22.160ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103252024-01-10T17:59:22.160ZLoading write count information from file "/tmp/test_repair_verify.data"
103262024-01-10T17:59:22.160ZRun Repair workload
103272024-01-10T17:59:22.160Z01/30 Write block 538 len 36864 data: 7 6 8 8 5 6 7 8 8
103282024-01-10T17:59:22.160Z02/30 Write block 389 len 32768 data: 5 7 7 7 6 8 8 7
103292024-01-10T17:59:22.160Z03/30 Write block 339 len 32768 data: 10 9 9 12 11 9 9 9
103302024-01-10T17:59:22.160Z04/30 Read block 264 len 40960
103312024-01-10T17:59:22.164Z05/30 Write block 277 len 16384 data: 9 8 10 9
103322024-01-10T17:59:22.164Z06/30 Write block 111 len 32768 data: 10 12 12 9 7 7 7 6
103332024-01-10T17:59:22.164Z07/30 Flush
103342024-01-10T17:59:22.169Z08/30 Write block 95 len 4096 data: 10
103352024-01-10T17:59:22.169Z09/30 Write block 172 len 4096 data: 6
103362024-01-10T17:59:22.169Z10/30 Write block 242 len 28672 data: 6 7 10 9 6 6 7
103372024-01-10T17:59:22.169Z11/30 Write block 399 len 36864 data: 9 7 8 7 9 10 12 13 12
103382024-01-10T17:59:22.172Z12/30 Write block 145 len 16384 data: 6 7 9 9
103392024-01-10T17:59:22.172Z13/30 Flush
103402024-01-10T17:59:22.175Z14/30 Write block 56 len 36864 data: 9 8 10 10 9 8 7 7 8
103412024-01-10T17:59:22.175Z15/30 Write block 347 len 32768 data: 10 7 6 5 5 6 5 5
103422024-01-10T17:59:22.178Z16/30 Flush
103432024-01-10T17:59:22.181Z17/30 Write block 542 len 40960 data: 6 7 8 9 9 13 13 14 13 13
103442024-01-10T17:59:22.181Z18/30 Read block 3 len 32768
103452024-01-10T17:59:22.185Z19/30 Write block 550 len 32768 data: 14 14 15 12 11 9 7 7
103462024-01-10T17:59:22.185Z20/30 Read block 344 len 4096
103472024-01-10T17:59:22.189Z21/30 Write block 196 len 28672 data: 12 12 13 13 12 10 9
103482024-01-10T17:59:22.189Z22/30 Read block 467 len 16384
103492024-01-10T17:59:22.193Z23/30 Read block 389 len 36864
103502024-01-10T17:59:22.196Z24/30 Write block 74 len 32768 data: 5 7 8 7 5 8 7 8
103512024-01-10T17:59:22.196Z25/30 Write block 164 len 12288 data: 12 13 13
103522024-01-10T17:59:22.196Z26/30 Write block 498 len 20480 data: 12 12 9 9 10
103532024-01-10T17:59:22.199Z27/30 Write block 123 len 4096 data: 6
103542024-01-10T17:59:22.199Z28/30 Read block 192 len 12288
103552024-01-10T17:59:22.199Z29/30 Read block 216 len 40960
103562024-01-10T17:59:22.202Z30/30 Read block 455 len 8192
103572024-01-10T17:59:22.206Z----------------------------------------------------------------
103582024-01-10T17:59:22.206Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30
103592024-01-10T17:59:22.206ZStates: Active Active Active
103602024-01-10T17:59:22.206ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
103612024-01-10T17:59:22.206Z 1 Acked 1000 Write 9 Done Done Done false
103622024-01-10T17:59:22.206Z 2 Acked 1001 Write 8 Done Done Done false
103632024-01-10T17:59:22.206Z 3 Acked 1002 Write 8 Done Done Sent false
103642024-01-10T17:59:22.206Z 4 Acked 1003 Read 10 Done Done Sent false
103652024-01-10T17:59:22.206Z 5 Acked 1004 Write 4 Done Done Sent false
103662024-01-10T17:59:22.206Z 6 Acked 1005 Write 8 Done Done Sent false
103672024-01-10T17:59:22.206Z 7 Acked 1006 Flush 0 Done Done Sent false
103682024-01-10T17:59:22.206Z 8 Acked 1007 Write 1 Done Done Sent false
103692024-01-10T17:59:22.206Z 9 Acked 1008 Write 1 Done Done Sent false
103702024-01-10T17:59:22.206Z 10 Acked 1009 Write 7 Done Done Sent false
103712024-01-10T17:59:22.206Z 11 Acked 1010 Write 9 Done Done Sent false
103722024-01-10T17:59:22.206Z 12 Acked 1011 Write 4 Done Done Sent false
103732024-01-10T17:59:22.206Z 13 Acked 1012 Flush 0 Done Done Sent false
103742024-01-10T17:59:22.207Z 14 Acked 1013 Write 9 Done Done Sent false
103752024-01-10T17:59:22.207Z 15 Acked 1014 Write 8 Done Done Sent false
103762024-01-10T17:59:22.207Z 16 Acked 1015 Flush 0 Done Done Sent false
103772024-01-10T17:59:22.207Z 17 Acked 1016 Write 10 Done Done Sent false
103782024-01-10T17:59:22.207Z 18 Acked 1017 Read 8 Done Done Sent false
103792024-01-10T17:59:22.208Z 19 Acked 1018 Write 8 Done Done Sent false
103802024-01-10T17:59:22.208Z 20 Acked 1019 Read 1 Done Done Sent false
103812024-01-10T17:59:22.208Z 21 Acked 1020 Write 7 Done Done Sent false
103822024-01-10T17:59:22.208Z 22 Acked 1021 Read 4 Done Done Sent false
103832024-01-10T17:59:22.208Z 23 Acked 1022 Read 9 Done Done Sent false
103842024-01-10T17:59:22.208Z 24 Acked 1023 Write 8 Done Done Sent false
103852024-01-10T17:59:22.208Z 25 Acked 1024 Write 3 Done Done Sent false
103862024-01-10T17:59:22.208Z 26 Acked 1025 Write 5 Done Done Sent false
103872024-01-10T17:59:22.208Z 27 Acked 1026 Write 1 Done Done Sent false
103882024-01-10T17:59:22.208Z 28 Acked 1027 Read 3 Done Done Sent false
103892024-01-10T17:59:22.208Z 29 Acked 1028 Read 10 Done Done Sent false
103902024-01-10T17:59:22.208Z 30 Acked 1029 Read 2 Done Done Sent false
103912024-01-10T17:59:22.208Z STATES DS:0 DS:1 DS:2 TOTAL
103922024-01-10T17:59:22.208Z New 0 0 0 0
103932024-01-10T17:59:22.208Z Sent 0 0 28 28
103942024-01-10T17:59:22.208Z Done 30 30 2 62
103952024-01-10T17:59:22.208Z Skipped 0 0 0 0
103962024-01-10T17:59:22.208Z Error 0 0 0 0
103972024-01-10T17:59:22.208ZLast Flush: 1015 1015 0
103982024-01-10T17:59:22.208ZDownstairs last five completed:
103992024-01-10T17:59:22.209ZUpstairs last five completed: 30 29 28 27 26
104002024-01-10T17:59:22.209ZWrote out file "/tmp/test_repair_verify.data"
104012024-01-10T17:59:22.213Z
104022024-01-10T17:59:22.218ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104032024-01-10T17:59:22.243ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104042024-01-10T17:59:22.245Z 0 000-019 91 91 91 174 174 174 F F F
104052024-01-10T17:59:22.245Z 1 020-039 89 89 89 169 169 169 F F F
104062024-01-10T17:59:22.245Z 2 040-059 96 96 92 185 185 176 F F F <---
104072024-01-10T17:59:22.245Z 3 060-079 96 96 92 185 185 176 T T F <---
104082024-01-10T17:59:22.245Z 4 080-099 96 96 87 184 184 163 T T F <---
104092024-01-10T17:59:22.251Z 5 100-119 96 96 92 183 183 176 F F F <---
104102024-01-10T17:59:22.251Z 6 120-139 95 95 95 182 182 182 T T F <---
104112024-01-10T17:59:22.251Z 7 140-159 96 96 93 184 184 177 F F F <---
104122024-01-10T17:59:22.251Z 8 160-179 96 96 81 184 184 150 T T F <---
104132024-01-10T17:59:22.251Z 9 180-199 94 94 94 180 180 180 T T F <---
104142024-01-10T17:59:22.251Z 10 200-219 90 90 90 172 172 172 T T F <---
104152024-01-10T17:59:22.251Z 11 220-239 86 86 86 160 160 160 F F F
104162024-01-10T17:59:22.251Z 12 240-259 96 96 86 184 184 160 F F F <---
104172024-01-10T17:59:22.251Z 13 260-279 96 96 90 183 183 170 F F F <---
104182024-01-10T17:59:22.251Z 14 280-299 96 96 94 183 183 179 F F F <---
104192024-01-10T17:59:22.251Z 15 300-319 94 94 94 180 180 180 F F F
104202024-01-10T17:59:22.251Z 16 320-339 96 96 94 183 183 181 F F F <---
104212024-01-10T17:59:22.252Z 17 340-359 96 96 94 185 185 181 F F F <---
104222024-01-10T17:59:22.252Z 18 360-379 95 95 95 182 182 182 F F F
104232024-01-10T17:59:22.252Z 19 380-399 96 96 95 184 184 182 F F T <---
104242024-01-10T17:59:22.252Z 20 400-419 96 96 93 184 184 177 F F F <---
104252024-01-10T17:59:22.252Z 21 420-439 92 92 92 175 175 175 F F F
104262024-01-10T17:59:22.252Z 22 440-459 92 92 92 176 176 176 F F F
104272024-01-10T17:59:22.252Z 23 460-479 77 77 77 142 142 142 F F F
104282024-01-10T17:59:22.252Z 24 480-499 90 90 90 173 173 173 T T F <---
104292024-01-10T17:59:22.252Z 25 500-519 94 94 94 181 181 181 T T F <---
104302024-01-10T17:59:22.252Z 26 520-539 96 96 90 183 183 172 F F T <---
104312024-01-10T17:59:22.253Z 27 540-559 96 96 95 183 183 182 T T T <---
104322024-01-10T17:59:22.253Z 28 560-579 85 85 85 159 159 159 F F F
104332024-01-10T17:59:22.253Z 29 580-599 86 86 86 162 162 162 F F F
104342024-01-10T17:59:22.253ZMax gen: 96, Max flush: 185
104352024-01-10T17:59:22.253ZError: Difference in extent metadata found!
104362024-01-10T17:59:22.253ZA Difference in extent metadata is expected here
104372024-01-10T17:59:22.253ZOn loop 48
104382024-01-10T17:59:22.253Z
104392024-01-10T17:59:22.257ZVerifying data now
104402024-01-10T17:59:29.088ZNo change after verify
104412024-01-10T17:59:29.089ZLoop: 48 Downstairs dump after verify (and repair):
104422024-01-10T17:59:29.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104432024-01-10T17:59:29.110Z 0 000-019 91 91 91 174 174 174 F F F
104442024-01-10T17:59:29.110Z 1 020-039 89 89 89 169 169 169 F F F
104452024-01-10T17:59:29.110Z 2 040-059 96 96 96 185 185 185 F F F
104462024-01-10T17:59:29.110Z 3 060-079 97 97 97 186 186 186 F F F
104472024-01-10T17:59:29.110Z 4 080-099 97 97 97 186 186 186 F F F
104482024-01-10T17:59:29.111Z 5 100-119 96 96 96 183 183 183 F F F
104492024-01-10T17:59:29.111Z 6 120-139 97 97 97 186 186 186 F F F
104502024-01-10T17:59:29.111Z 7 140-159 96 96 96 184 184 184 F F F
104512024-01-10T17:59:29.114Z 8 160-179 97 97 97 186 186 186 F F F
104522024-01-10T17:59:29.115Z 9 180-199 97 97 97 186 186 186 F F F
104532024-01-10T17:59:29.115Z 10 200-219 97 97 97 186 186 186 F F F
104542024-01-10T17:59:29.115Z 11 220-239 86 86 86 160 160 160 F F F
104552024-01-10T17:59:29.115Z 12 240-259 96 96 96 184 184 184 F F F
104562024-01-10T17:59:29.115Z 13 260-279 96 96 96 183 183 183 F F F
104572024-01-10T17:59:29.115Z 14 280-299 96 96 96 183 183 183 F F F
104582024-01-10T17:59:29.115Z 15 300-319 94 94 94 180 180 180 F F F
104592024-01-10T17:59:29.115Z 16 320-339 96 96 96 183 183 183 F F F
104602024-01-10T17:59:29.115Z 17 340-359 96 96 96 185 185 185 F F F
104612024-01-10T17:59:29.115Z 18 360-379 95 95 95 182 182 182 F F F
104622024-01-10T17:59:29.115Z 19 380-399 96 96 96 184 184 184 F F F
104632024-01-10T17:59:29.116Z 20 400-419 96 96 96 184 184 184 F F F
104642024-01-10T17:59:29.116Z 21 420-439 92 92 92 175 175 175 F F F
104652024-01-10T17:59:29.116Z 22 440-459 92 92 92 176 176 176 F F F
104662024-01-10T17:59:29.116Z 23 460-479 77 77 77 142 142 142 F F F
104672024-01-10T17:59:29.116Z 24 480-499 97 97 97 186 186 186 F F F
104682024-01-10T17:59:29.116Z 25 500-519 97 97 97 186 186 186 F F F
104692024-01-10T17:59:29.116Z 26 520-539 96 96 96 183 183 183 F F F
104702024-01-10T17:59:29.116Z 27 540-559 97 97 97 186 186 186 F F F
104712024-01-10T17:59:29.116Z 28 560-579 85 85 85 159 159 159 F F F
104722024-01-10T17:59:29.116Z 29 580-599 86 86 86 162 162 162 F F F
104732024-01-10T17:59:29.116ZMax gen: 97, Max flush: 186
104742024-01-10T17:59:29.120Z
104752024-01-10T17:59:29.120ZBegin loop 49
104762024-01-10T17:59:29.121ZDownstairs to restart: 2
104772024-01-10T17:59:29.146Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.14933308Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547}
104782024-01-10T17:59:29.146Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.149929999Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547}
104792024-01-10T17:59:29.146Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.149976165Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547}
104802024-01-10T17:59:29.151Z{"msg":"Crucible 2e0e3c14-8e3e-48fe-b5da-bd2715161460 has session id: 1eec05ec-d10c-4c97-b2d3-0994157de40e","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150082306Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104812024-01-10T17:59:29.151Z{"msg":"Crucible runtime is spawned
104822024-01-10T17:59:29.151ZUpstairs opts: Upstairs UUID: 2e0e3c14-8e3e-48fe-b5da-bd2715161460, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150197481Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104832024-01-10T17:59:29.152Z{"msg":"Crucible stats registered with UUID: 2e0e3c14-8e3e-48fe-b5da-bd2715161460","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150265171Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104842024-01-10T17:59:29.152Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150337726Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547}
104852024-01-10T17:59:29.152Z{"msg":"2e0e3c14-8e3e-48fe-b5da-bd2715161460 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150495297Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104862024-01-10T17:59:29.152Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150552709Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"0","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104872024-01-10T17:59:29.152Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150578151Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"0","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104882024-01-10T17:59:29.152Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150600206Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"1","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104892024-01-10T17:59:29.152Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150643031Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"1","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104902024-01-10T17:59:29.152Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150666333Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"2","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104912024-01-10T17:59:29.152Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150695038Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"2","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104922024-01-10T17:59:29.152Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150763081Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104932024-01-10T17:59:29.153Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150834094Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104942024-01-10T17:59:29.153Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.150882196Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104952024-01-10T17:59:29.153Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.151101436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104962024-01-10T17:59:29.153Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.151198318Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104972024-01-10T17:59:29.153Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.151228961Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104982024-01-10T17:59:29.153Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.152579359Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"0","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
104992024-01-10T17:59:29.153Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.152650908Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"0","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105002024-01-10T17:59:29.153Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.152678443Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"1","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105012024-01-10T17:59:29.153Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.152708398Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"1","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105022024-01-10T17:59:29.153Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.153261652Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"2","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105032024-01-10T17:59:29.154Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.153347216Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"2","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105042024-01-10T17:59:29.154Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.153550325Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"0","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105052024-01-10T17:59:29.155Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.153619506Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"1","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105062024-01-10T17:59:29.160Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.154363398Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"1","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105072024-01-10T17:59:29.160Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.154432318Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"1","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105082024-01-10T17:59:29.160Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.15445866Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105092024-01-10T17:59:29.162Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.15455491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"2","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105102024-01-10T17:59:29.162Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.154607553Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"0","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105112024-01-10T17:59:29.162Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.154632561Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"0","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105122024-01-10T17:59:29.162Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.154655038Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105132024-01-10T17:59:29.165Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155372239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"2","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105142024-01-10T17:59:29.165Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155514665Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"2","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105152024-01-10T17:59:29.165Z{"msg":"[0]R flush_numbers[0..12]: [174, 169, 185, 186, 186, 183, 186, 184, 186, 186, 186, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155560224Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105162024-01-10T17:59:29.165Z{"msg":"[0]R generation[0..12]: [91, 89, 96, 97, 97, 96, 97, 96, 97, 97, 97, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155607287Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105172024-01-10T17:59:29.165Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.15563385Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105182024-01-10T17:59:29.165Z{"msg":"[1]R flush_numbers[0..12]: [174, 169, 185, 186, 186, 183, 186, 184, 186, 186, 186, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155675326Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105192024-01-10T17:59:29.165Z{"msg":"[1]R generation[0..12]: [91, 89, 96, 97, 97, 96, 97, 96, 97, 97, 97, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155699734Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105202024-01-10T17:59:29.165Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155723301Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105212024-01-10T17:59:29.166Z{"msg":"[2]R flush_numbers[0..12]: [174, 169, 185, 186, 186, 183, 186, 184, 186, 186, 186, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155746541Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105222024-01-10T17:59:29.168Z{"msg":"[2]R generation[0..12]: [91, 89, 96, 97, 97, 96, 97, 96, 97, 97, 97, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155771004Z","hostname":"Wait for a query_work_queue command to finish before sending IO
105232024-01-10T17:59:29.168Zip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105242024-01-10T17:59:29.168Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155860666Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105252024-01-10T17:59:29.168Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.15588726Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105262024-01-10T17:59:29.168Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155930633Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105272024-01-10T17:59:29.168Z{"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155953878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105282024-01-10T17:59:29.168Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.155993654Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"":"downstairs","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105292024-01-10T17:59:29.169Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.156017007Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105302024-01-10T17:59:29.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.156038013Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"0","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105312024-01-10T17:59:29.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.156060025Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"1","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105322024-01-10T17:59:29.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.156081578Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"client":"2","":"downstairs","session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105332024-01-10T17:59:29.171Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.156121038Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105342024-01-10T17:59:29.172Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.156142494Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105352024-01-10T17:59:29.172Z{"msg":"2e0e3c14-8e3e-48fe-b5da-bd2715161460 is now active with session: 1eec05ec-d10c-4c97-b2d3-0994157de40e","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.156182411Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105362024-01-10T17:59:29.172Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.156203266Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547,"session_id":"1eec05ec-d10c-4c97-b2d3-0994157de40e"}
105372024-01-10T17:59:29.172Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:29.156223374Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1547}
105382024-01-10T17:59:29.172ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105392024-01-10T17:59:29.172ZLoading write count information from file "/tmp/test_repair_verify.data"
105402024-01-10T17:59:29.172ZRun Repair workload
105412024-01-10T17:59:29.172Z01/30 Write block 9 len 12288 data: 6 7 8
105422024-01-10T17:59:29.172Z02/30 Write block 289 len 16384 data: 9 11 12 12
105432024-01-10T17:59:29.172Z03/30 Read block 593 len 8192
105442024-01-10T17:59:29.178Z04/30 Write block 31 len 36864 data: 11 11 10 9 9 9 8 8 5
105452024-01-10T17:59:29.178Z05/30 Write block 201 len 20480 data: 11 10 8 6 5
105462024-01-10T17:59:29.178Z06/30 Flush
105472024-01-10T17:59:29.178Z07/30 Write block 335 len 8192 data: 7 8
105482024-01-10T17:59:29.178Z08/30 Read block 200 len 12288
105492024-01-10T17:59:29.179Z09/30 Flush
105502024-01-10T17:59:29.179Z10/30 Write block 520 len 36864 data: 9 9 8 4 3 6 9 8 9
105512024-01-10T17:59:29.179Z11/30 Read block 179 len 20480
105522024-01-10T17:59:29.179Z12/30 Write block 46 len 24576 data: 5 4 9 11 12 9
105532024-01-10T17:59:29.179Z13/30 Write block 324 len 24576 data: 10 10 7 7 7 6
105542024-01-10T17:59:29.179Z14/30 Write block 487 len 16384 data: 6 7 8 10
105552024-01-10T17:59:29.193Z15/30 Write block 302 len 12288 data: 8 9 11
105562024-01-10T17:59:29.194Z16/30 Write block 550 len 28672 data: 15 15 16 13 12 10 8
105572024-01-10T17:59:29.194Z17/30 Write block 45 len 20480 data: 7 6 5 10 12
105582024-01-10T17:59:29.194Z18/30 Read block 125 len 32768
105592024-01-10T17:59:29.197Z19/30 Write block 94 len 16384 data: 7 11 10 10
105602024-01-10T17:59:29.198Z20/30 Write block 404 len 20480 data: 11 13 14 13 10
105612024-01-10T17:59:29.198Z21/30 Read block 405 len 40960
105622024-01-10T17:59:29.198Z22/30 Read block 345 len 40960
105632024-01-10T17:59:29.198Z23/30 Read block 377 len 4096
105642024-01-10T17:59:29.198Z24/30 Read block 50 len 20480
105652024-01-10T17:59:29.198Z25/30 Write block 552 len 40960 data: 17 14 13 11 9 8 6 6 7 8
105662024-01-10T17:59:29.198Z26/30 Write block 363 len 16384 data: 9 8 7 7
105672024-01-10T17:59:29.202Z27/30 Read block 376 len 32768
105682024-01-10T17:59:29.202Z28/30 Read block 74 len 4096
105692024-01-10T17:59:29.205Z29/30 Read block 177 len 32768
105702024-01-10T17:59:29.205Z30/30 Read block 314 len 4096
105712024-01-10T17:59:29.209Z----------------------------------------------------------------
105722024-01-10T17:59:29.209Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
105732024-01-10T17:59:29.209ZStates: Active Active Active
105742024-01-10T17:59:29.209ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
105752024-01-10T17:59:29.210Z 1 Acked 1000 Write 3 Done Done Sent false
105762024-01-10T17:59:29.210Z 2 Acked 1001 Write 4 Done Done Sent false
105772024-01-10T17:59:29.210Z 3 Acked 1002 Read 2 Done Done Sent false
105782024-01-10T17:59:29.210Z 4 Acked 1003 Write 9 Done Done Sent false
105792024-01-10T17:59:29.210Z 5 Acked 1004 Write 5 Done Done Sent false
105802024-01-10T17:59:29.210Z 6 Acked 1005 Flush 0 Done Done Sent false
105812024-01-10T17:59:29.210Z 7 Acked 1006 Write 2 Done Done Sent false
105822024-01-10T17:59:29.210Z 8 Acked 1007 Read 3 Done Done Sent false
105832024-01-10T17:59:29.210Z 9 Acked 1008 Flush 0 Done Done Sent false
105842024-01-10T17:59:29.210Z 10 Acked 1009 Write 9 Done Done Sent false
105852024-01-10T17:59:29.210Z 11 Acked 1010 Read 5 Done Done Sent false
105862024-01-10T17:59:29.210Z 12 Acked 1011 Write 6 Done Done Sent false
105872024-01-10T17:59:29.211Z 13 Acked 1012 Write 6 Done Done Sent false
105882024-01-10T17:59:29.211Z 14 Acked 1013 Write 4 Done Done Sent false
105892024-01-10T17:59:29.211Z 15 Acked 1014 Write 3 Done Done Sent false
105902024-01-10T17:59:29.211Z 16 Acked 1015 Write 7 Done Done Sent false
105912024-01-10T17:59:29.211Z 17 Acked 1016 Write 5 Done Done Sent false
105922024-01-10T17:59:29.211Z 18 Acked 1017 Read 8 Done Done Sent false
105932024-01-10T17:59:29.211Z 19 Acked 1018 Write 4 Done Done Sent false
105942024-01-10T17:59:29.211Z 20 Acked 1019 Write 5 Done Done Sent false
105952024-01-10T17:59:29.211Z 21 Acked 1020 Read 10 Done Done Sent false
105962024-01-10T17:59:29.211Z 22 Acked 1021 Read 10 Done Done Sent false
105972024-01-10T17:59:29.211Z 23 Acked 1022 Read 1 Done Done Sent false
105982024-01-10T17:59:29.211Z 24 Acked 1023 Read 5 Done Done Sent false
105992024-01-10T17:59:29.212Z 25 Acked 1024 Write 10 Done Done Sent false
106002024-01-10T17:59:29.212Z 26 Acked 1025 Write 4 Done Done Sent false
106012024-01-10T17:59:29.213Z 27 Acked 1026 Read 8 Done Done Sent false
106022024-01-10T17:59:29.213Z 28 Acked 1027 Read 1 Done Done Sent false
106032024-01-10T17:59:29.213Z 29 Acked 1028 Read 8 Done Done Sent false
106042024-01-10T17:59:29.213Z 30 Acked 1029 Read 1 Done Sent Sent false
106052024-01-10T17:59:29.213Z STATES DS:0 DS:1 DS:2 TOTAL
106062024-01-10T17:59:29.213Z New 0 0 0 0
106072024-01-10T17:59:29.213Z Sent 0 1 30 31
106082024-01-10T17:59:29.213Z Done 30 29 0 59
106092024-01-10T17:59:29.213Z Skipped 0 0 0 0
106102024-01-10T17:59:29.213Z Error 0 0 0 0
106112024-01-10T17:59:29.213ZLast Flush: 1008 1008 0
106122024-01-10T17:59:29.214ZDownstairs last five completed:
106132024-01-10T17:59:29.214ZUpstairs last five completed: 30 29 28 27 26
106142024-01-10T17:59:29.215ZWrote out file "/tmp/test_repair_verify.data"
106152024-01-10T17:59:29.219Z
106162024-01-10T17:59:29.223ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106172024-01-10T17:59:29.249ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106182024-01-10T17:59:29.249Z 0 000-019 98 98 91 187 187 174 F F F <---
106192024-01-10T17:59:29.250Z 1 020-039 98 98 89 187 187 169 F F F <---
106202024-01-10T17:59:29.250Z 2 040-059 96 96 96 185 185 185 T T F <---
106212024-01-10T17:59:29.250Z 3 060-079 97 97 97 186 186 186 F F F
106222024-01-10T17:59:29.250Z 4 080-099 97 97 97 186 186 186 T T F <---
106232024-01-10T17:59:29.250Z 5 100-119 96 96 96 183 183 183 F F F
106242024-01-10T17:59:29.250Z 6 120-139 97 97 97 186 186 186 F F F
106252024-01-10T17:59:29.250Z 7 140-159 96 96 96 184 184 184 F F F
106262024-01-10T17:59:29.250Z 8 160-179 97 97 97 186 186 186 F F F
106272024-01-10T17:59:29.254Z 9 180-199 97 97 97 186 186 186 F F F
106282024-01-10T17:59:29.254Z 10 200-219 98 98 97 187 187 186 F F F <---
106292024-01-10T17:59:29.255Z 11 220-239 86 86 86 160 160 160 F F F
106302024-01-10T17:59:29.255Z 12 240-259 96 96 96 184 184 184 F F F
106312024-01-10T17:59:29.255Z 13 260-279 96 96 96 183 183 183 F F F
106322024-01-10T17:59:29.255Z 14 280-299 98 98 96 187 187 183 F F F <---
106332024-01-10T17:59:29.255Z 15 300-319 94 94 94 180 180 180 T T F <---
106342024-01-10T17:59:29.255Z 16 320-339 98 98 96 188 188 183 T T F <---
106352024-01-10T17:59:29.255Z 17 340-359 96 96 96 185 185 185 F F F
106362024-01-10T17:59:29.255Z 18 360-379 95 95 95 182 182 182 T T F <---
106372024-01-10T17:59:29.255Z 19 380-399 96 96 96 184 184 184 F F F
106382024-01-10T17:59:29.255Z 20 400-419 96 96 96 184 184 184 T T F <---
106392024-01-10T17:59:29.255Z 21 420-439 92 92 92 175 175 175 F F F
106402024-01-10T17:59:29.255Z 22 440-459 92 92 92 176 176 176 F F F
106412024-01-10T17:59:29.256Z 23 460-479 77 77 77 142 142 142 F F F
106422024-01-10T17:59:29.256Z 24 480-499 97 97 97 186 186 186 T T F <---
106432024-01-10T17:59:29.256Z 25 500-519 97 97 97 186 186 186 F F F
106442024-01-10T17:59:29.256Z 26 520-539 96 96 96 183 183 183 T T F <---
106452024-01-10T17:59:29.256Z 27 540-559 97 97 97 186 186 186 T T F <---
106462024-01-10T17:59:29.256Z 28 560-579 85 85 85 159 159 159 T T F <---
106472024-01-10T17:59:29.258Z 29 580-599 86 86 86 162 162 162 F F F
106482024-01-10T17:59:29.258ZMax gen: 98, Max flush: 188
106492024-01-10T17:59:29.258ZError: Difference in extent metadata found!
106502024-01-10T17:59:29.262ZA Difference in extent metadata is expected here
106512024-01-10T17:59:29.262ZOn loop 49
106522024-01-10T17:59:29.262Z
106532024-01-10T17:59:29.262ZVerifying data now
106542024-01-10T17:59:34.661ZNo change after verify
106552024-01-10T17:59:34.661ZLoop: 49 Downstairs dump after verify (and repair):
106562024-01-10T17:59:34.683ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106572024-01-10T17:59:34.683Z 0 000-019 98 98 98 187 187 187 F F F
106582024-01-10T17:59:34.685Z 1 020-039 98 98 98 187 187 187 F F F
106592024-01-10T17:59:34.685Z 2 040-059 99 99 99 189 189 189 F F F
106602024-01-10T17:59:34.685Z 3 060-079 97 97 97 186 186 186 F F F
106612024-01-10T17:59:34.685Z 4 080-099 99 99 99 189 189 189 F F F
106622024-01-10T17:59:34.685Z 5 100-119 96 96 96 183 183 183 F F F
106632024-01-10T17:59:34.685Z 6 120-139 97 97 97 186 186 186 F F F
106642024-01-10T17:59:34.685Z 7 140-159 96 96 96 184 184 184 F F F
106652024-01-10T17:59:34.689Z 8 160-179 97 97 97 186 186 186 F F F
106662024-01-10T17:59:34.689Z 9 180-199 97 97 97 186 186 186 F F F
106672024-01-10T17:59:34.690Z 10 200-219 98 98 98 187 187 187 F F F
106682024-01-10T17:59:34.690Z 11 220-239 86 86 86 160 160 160 F F F
106692024-01-10T17:59:34.690Z 12 240-259 96 96 96 184 184 184 F F F
106702024-01-10T17:59:34.690Z 13 260-279 96 96 96 183 183 183 F F F
106712024-01-10T17:59:34.690Z 14 280-299 98 98 98 187 187 187 F F F
106722024-01-10T17:59:34.690Z 15 300-319 99 99 99 189 189 189 F F F
106732024-01-10T17:59:34.690Z 16 320-339 99 99 99 189 189 189 F F F
106742024-01-10T17:59:34.690Z 17 340-359 96 96 96 185 185 185 F F F
106752024-01-10T17:59:34.690Z 18 360-379 99 99 99 189 189 189 F F F
106762024-01-10T17:59:34.690Z 19 380-399 96 96 96 184 184 184 F F F
106772024-01-10T17:59:34.690Z 20 400-419 99 99 99 189 189 189 F F F
106782024-01-10T17:59:34.690Z 21 420-439 92 92 92 175 175 175 F F F
106792024-01-10T17:59:34.690Z 22 440-459 92 92 92 176 176 176 F F F
106802024-01-10T17:59:34.690Z 23 460-479 77 77 77 142 142 142 F F F
106812024-01-10T17:59:34.690Z 24 480-499 99 99 99 189 189 189 F F F
106822024-01-10T17:59:34.690Z 25 500-519 97 97 97 186 186 186 F F F
106832024-01-10T17:59:34.690Z 26 520-539 99 99 99 189 189 189 F F F
106842024-01-10T17:59:34.690Z 27 540-559 99 99 99 189 189 189 F F F
106852024-01-10T17:59:34.690Z 28 560-579 99 99 99 189 189 189 F F F
106862024-01-10T17:59:34.690Z 29 580-599 86 86 86 162 162 162 F F F
106872024-01-10T17:59:34.690ZMax gen: 99, Max flush: 189
106882024-01-10T17:59:34.690Z
106892024-01-10T17:59:34.690ZBegin loop 50
106902024-01-10T17:59:34.690ZDownstairs to restart: 0
106912024-01-10T17:59:34.719Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.722563944Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555}
106922024-01-10T17:59:34.719Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723143661Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555}
106932024-01-10T17:59:34.719Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v"Crucible runtime is spawned
106942024-01-10T17:59:34.722Z:0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723258938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555}
106952024-01-10T17:59:34.722Z{"msg":"Crucible 0313ec42-3820-458b-a8b8-ac0707a70684 has session id: f398f335-1cb4-49fe-ac6f-7925f6e9a27d","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723307372Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
106962024-01-10T17:59:34.723Z{"msg":"Upstairs opts: Upstairs UUID: 0313ec42-3820-458b-a8b8-ac0707a70684, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.72335598Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
106972024-01-10T17:59:34.723Z{"msg":"Crucible stats registered with UUID: 0313ec42-3820-458b-a8b8-ac0707a70684","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723397477Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
106982024-01-10T17:59:34.723Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723434578Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555}
106992024-01-10T17:59:34.723Z{"msg":"0313ec42-3820-458b-a8b8-ac0707a70684 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723607145Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107002024-01-10T17:59:34.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723678725Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"0","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107012024-01-10T17:59:34.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723724078Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"0","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107022024-01-10T17:59:34.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723761443Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"1","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107032024-01-10T17:59:34.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723801339Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"1","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107042024-01-10T17:59:34.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723844158Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"2","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107052024-01-10T17:59:34.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723886529Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"2","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107062024-01-10T17:59:34.723Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723928547Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107072024-01-10T17:59:34.723Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.723971287Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107082024-01-10T17:59:34.723Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.724013252Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107092024-01-10T17:59:34.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.724146215Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107102024-01-10T17:59:34.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.72420763Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107112024-01-10T17:59:34.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.72425038Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107122024-01-10T17:59:34.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.725640461Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"1","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107132024-01-10T17:59:34.723Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.72579529Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"1","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107142024-01-10T17:59:34.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.72585267Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"2","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107152024-01-10T17:59:34.723Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.725895726Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"2","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107162024-01-10T17:59:34.730Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.726572539Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"2","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107172024-01-10T17:59:34.730Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.726763877Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"1","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107182024-01-10T17:59:34.730Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.727351519Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"2","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107192024-01-10T17:59:34.730Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.727510086Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"2","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107202024-01-10T17:59:34.730Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.727578868Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107212024-01-10T17:59:34.730Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.727603897Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"1","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107222024-01-10T17:59:34.730Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.727651241Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"1","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107232024-01-10T17:59:34.730Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.727677899Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107242024-01-10T17:59:34.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.727717453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"0","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107252024-01-10T17:59:34.731Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.727741134Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"0","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107262024-01-10T17:59:34.731Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.728480302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"0","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107272024-01-10T17:59:34.731Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729136688Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"0","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107282024-01-10T17:59:34.731Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729214952Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"0","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107292024-01-10T17:59:34.731Z{"msg":"[0]R flush_numbers[0..12]: [187, 187, 189, 186, 189, 183, 186, 184, 186, 186, 187, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729261872Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107302024-01-10T17:59:34.731Z{"msg":"[0]R generation[0..12]: [98, 98, 99, 97, 99, 96, 97, 96, 97, 97, 98, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729322128Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107312024-01-10T17:59:34.731Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729355272Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107322024-01-10T17:59:34.731Z{"msg":"[1]R flush_numbers[0..12]: [187, 187, 189, 186, 189, 183, 186, 184, 186, 186, 187, 160]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
107332024-01-10T17:59:34.731Z,"time":"2024-01-10T17:59:34.72945107Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107342024-01-10T17:59:34.732Z{"msg":"[1]R generation[0..12]: [98, 98, 99, 97, 99, 96, 97, 96, 97, 97, 98, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729565039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107352024-01-10T17:59:34.732Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.72959493Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107362024-01-10T17:59:34.732Z{"msg":"[2]R flush_numbers[0..12]: [187, 187, 189, 186, 189, 183, 186, 184, 186, 186, 187, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729637639Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107372024-01-10T17:59:34.732Z{"msg":"[2]R generation[0..12]: [98, 98, 99, 97, 99, 96, 97, 96, 97, 97, 98, 86]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729661779Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107382024-01-10T17:59:34.733Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729703721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107392024-01-10T17:59:34.733Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729726848Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107402024-01-10T17:59:34.733Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729775324Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107412024-01-10T17:59:34.733Z{"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729797796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107422024-01-10T17:59:34.733Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729837598Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"":"downstairs","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107432024-01-10T17:59:34.733Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729860478Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107442024-01-10T17:59:34.733Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729898692Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"0","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107452024-01-10T17:59:34.733Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729921588Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"1","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107462024-01-10T17:59:34.733Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729963081Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"client":"2","":"downstairs","session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107472024-01-10T17:59:34.733Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.729995429Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107482024-01-10T17:59:34.733Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.730027867Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107492024-01-10T17:59:34.733Z{"msg":"0313ec42-3820-458b-a8b8-ac0707a70684 is now active with session: f398f335-1cb4-49fe-ac6f-7925f6e9a27d","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.730060291Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107502024-01-10T17:59:34.733Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.730089784Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555,"session_id":"f398f335-1cb4-49fe-ac6f-7925f6e9a27d"}
107512024-01-10T17:59:34.733Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:34.730120606Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1555}
107522024-01-10T17:59:34.733ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
107532024-01-10T17:59:34.733ZLoading write count information from file "/tmp/test_repair_verify.data"
107542024-01-10T17:59:34.733ZRun Repair workload
107552024-01-10T17:59:34.737Z01/30 Write block 460 len 36864 data: 6 6 6 7 7 5 7 8 8
107562024-01-10T17:59:34.737Z02/30 Write block 492 len 40960 data: 13 14 14 17 16 13 13 13 10 10
107572024-01-10T17:59:34.737Z03/30 Write block 408 len 24576 data: 11 10 12 13 10 10
107582024-01-10T17:59:34.737Z04/30 Write block 417 len 16384 data: 9 8 12 15
107592024-01-10T17:59:34.737Z05/30 Read block 11 len 28672
107602024-01-10T17:59:34.740Z06/30 Read block 85 len 28672
107612024-01-10T17:59:34.743Z07/30 Write block 534 len 32768 data: 11 10 10 8 8 7 9 9
107622024-01-10T17:59:34.743Z08/30 Write block 51 len 16384 data: 10 9 7 10
107632024-01-10T17:59:34.743Z09/30 Read block 379 len 12288
107642024-01-10T17:59:34.746Z10/30 Read block 569 len 8192
107652024-01-10T17:59:34.746Z11/30 Read block 503 len 12288
107662024-01-10T17:59:34.750Z12/30 Flush
107672024-01-10T17:59:34.755Z13/30 Write block 258 len 16384 data: 9 8 6 7
107682024-01-10T17:59:34.756Z14/30 Write block 233 len 16384 data: 2 2 2 4
107692024-01-10T17:59:34.756Z15/30 Write block 522 len 4096 data: 9
107702024-01-10T17:59:34.756Z16/30 Flush
107712024-01-10T17:59:34.759Z17/30 Flush
107722024-01-10T17:59:34.759Z18/30 Write block 218 len 4096 data: 7
107732024-01-10T17:59:34.762Z19/30 Write block 138 len 32768 data: 8 9 10 10 8 8 8 7
107742024-01-10T17:59:34.762Z20/30 Read block 300 len 36864
107752024-01-10T17:59:34.765Z21/30 Read block 47 len 8192
107762024-01-10T17:59:34.768Z22/30 Write block 262 len 12288 data: 5 6 6
107772024-01-10T17:59:34.768Z23/30 Read block 347 len 40960
107782024-01-10T17:59:34.771Z24/30 Read block 308 len 24576
107792024-01-10T17:59:34.774Z25/30 Write block 483 len 16384 data: 4 7 6 7
107802024-01-10T17:59:34.774Z26/30 Flush
107812024-01-10T17:59:34.778Z27/30 Write block 25 len 40960 data: 7 8 8 8 9 13 12 12 11 10
107822024-01-10T17:59:34.781Z28/30 Read block 165 len 40960
107832024-01-10T17:59:34.784Z29/30 Write block 353 len 28672 data: 6 6 4 5 6 8 7
107842024-01-10T17:59:34.787Z30/30 Read block 82 len 4096
107852024-01-10T17:59:34.788Z----------------------------------------------------------------
107862024-01-10T17:59:34.788Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
107872024-01-10T17:59:34.788ZStates: Active Active Active
107882024-01-10T17:59:34.788ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
107892024-01-10T17:59:34.788Z 1 Acked 1000 Write 9 Done Done Done false
107902024-01-10T17:59:34.788Z 2 Acked 1001 Write 10 Sent Done Done false
107912024-01-10T17:59:34.788Z 3 Acked 1002 Write 6 Sent Done Done false
107922024-01-10T17:59:34.788Z 4 Acked 1003 Write 4 Sent Done Done false
107932024-01-10T17:59:34.789Z 5 Acked 1004 Read 7 Sent Done Done false
107942024-01-10T17:59:34.789Z 6 Acked 1005 Read 7 Sent Done Done false
107952024-01-10T17:59:34.789Z 7 Acked 1006 Write 8 Sent Done Done false
107962024-01-10T17:59:34.789Z 8 Acked 1007 Write 4 Sent Done Done false
107972024-01-10T17:59:34.789Z 9 Acked 1008 Read 3 Sent Done Done false
107982024-01-10T17:59:34.789Z 10 Acked 1009 Read 2 Sent Done Done false
107992024-01-10T17:59:34.789Z 11 Acked 1010 Read 3 Sent Done Done false
108002024-01-10T17:59:34.789Z 12 Acked 1011 Flush 0 Sent Done Done false
108012024-01-10T17:59:34.789Z 13 Acked 1012 Write 4 Sent Done Done false
108022024-01-10T17:59:34.789Z 14 Acked 1013 Write 4 Sent Done Done false
108032024-01-10T17:59:34.789Z 15 Acked 1014 Write 1 Sent Done Done false
108042024-01-10T17:59:34.789Z 16 Acked 1015 Flush 0 Sent Done Done false
108052024-01-10T17:59:34.790Z 17 Acked 1016 Flush 0 Sent Done Done false
108062024-01-10T17:59:34.790Z 18 Acked 1017 Write 1 Sent Done Done false
108072024-01-10T17:59:34.790Z 19 Acked 1018 Write 8 Sent Done Done false
108082024-01-10T17:59:34.790Z 20 Acked 1019 Read 9 Sent Done Done false
108092024-01-10T17:59:34.790Z 21 Acked 1020 Read 2 Sent Done Done false
108102024-01-10T17:59:34.790Z 22 Acked 1021 Write 3 Sent Done Done false
108112024-01-10T17:59:34.790Z 23 Acked 1022 Read 10 Sent Done Done false
108122024-01-10T17:59:34.790Z 24 Acked 1023 Read 6 Sent Done Done false
108132024-01-10T17:59:34.790Z 25 Acked 1024 Write 4 Sent Done Done false
108142024-01-10T17:59:34.790Z 26 Acked 1025 Flush 0 Sent Done Done false
108152024-01-10T17:59:34.790Z 27 Acked 1026 Write 10 Sent Done Done false
108162024-01-10T17:59:34.791Z 28 Acked 1027 Read 10 Sent Done Done false
108172024-01-10T17:59:34.791Z 29 Acked 1028 Write 7 Sent Done Sent false
108182024-01-10T17:59:34.791Z 30 Acked 1029 Read 1 Sent Done Sent false
108192024-01-10T17:59:34.791Z STATES DS:0 DS:1 DS:2 TOTAL
108202024-01-10T17:59:34.791Z New 0 0 0 0
108212024-01-10T17:59:34.791Z Sent 29 0 2 31
108222024-01-10T17:59:34.791Z Done 1 30 28 59
108232024-01-10T17:59:34.791Z Skipped 0 0 0 0
108242024-01-10T17:59:34.793Z Error 0 0 0 0
108252024-01-10T17:59:34.793ZLast Flush: 0 1025 1025
108262024-01-10T17:59:34.793ZDownstairs last five completed:
108272024-01-10T17:59:34.793ZUpstairs last five completed: 30 29 28 27 26
108282024-01-10T17:59:34.793ZWrote out file "/tmp/test_repair_verify.data"
108292024-01-10T17:59:34.796Z
108302024-01-10T17:59:34.800ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108312024-01-10T17:59:34.824ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108322024-01-10T17:59:34.824Z 0 000-019 98 98 98 187 187 187 F F F
108332024-01-10T17:59:34.824Z 1 020-039 98 98 98 187 187 187 F T T <---
108342024-01-10T17:59:34.824Z 2 040-059 99 100 100 189 190 190 F F F <---
108352024-01-10T17:59:34.824Z 3 060-079 97 97 97 186 186 186 F F F
108362024-01-10T17:59:34.825Z 4 080-099 99 99 99 189 189 189 F F F
108372024-01-10T17:59:34.825Z 5 100-119 96 96 96 183 183 183 F F F
108382024-01-10T17:59:34.825Z 6 120-139 97 100 100 186 193 193 F F F <---
108392024-01-10T17:59:34.825Z 7 140-159 96 100 100 184 193 193 F F F <---
108402024-01-10T17:59:34.825Z 8 160-179 97 97 97 186 186 186 F F F
108412024-01-10T17:59:34.828Z 9 180-199 97 97 97 186 186 186 F F F
108422024-01-10T17:59:34.828Z 10 200-219 98 100 100 187 193 193 F F F <---
108432024-01-10T17:59:34.828Z 11 220-239 86 100 100 160 191 191 F F F <---
108442024-01-10T17:59:34.828Z 12 240-259 96 100 100 184 191 191 F F F <---
108452024-01-10T17:59:34.828Z 13 260-279 96 100 100 183 193 193 F F F <---
108462024-01-10T17:59:34.828Z 14 280-299 98 98 98 187 187 187 F F F
108472024-01-10T17:59:34.828Z 15 300-319 99 99 99 189 189 189 F F F
108482024-01-10T17:59:34.829Z 16 320-339 99 99 99 189 189 189 F F F
108492024-01-10T17:59:34.829Z 17 340-359 96 96 96 185 185 185 F T T <---
108502024-01-10T17:59:34.829Z 18 360-379 99 99 99 189 189 189 F F F
108512024-01-10T17:59:34.829Z 19 380-399 96 96 96 184 184 184 F F F
108522024-01-10T17:59:34.829Z 20 400-419 99 100 100 189 190 190 F F F <---
108532024-01-10T17:59:34.829Z 21 420-439 92 100 100 175 190 190 F F F <---
108542024-01-10T17:59:34.829Z 22 440-459 92 92 92 176 176 176 F F F
108552024-01-10T17:59:34.829Z 23 460-479 77 100 100 142 190 190 T F F <---
108562024-01-10T17:59:34.829Z 24 480-499 99 100 100 189 193 193 F F F <---
108572024-01-10T17:59:34.829Z 25 500-519 97 100 100 186 190 190 F F F <---
108582024-01-10T17:59:34.829Z 26 520-539 99 100 100 189 191 191 F F F <---
108592024-01-10T17:59:34.829Z 27 540-559 99 100 100 189 190 190 F F F <---
108602024-01-10T17:59:34.829Z 28 560-579 99 99 99 189 189 189 F F F
108612024-01-10T17:59:34.829Z 29 580-599 86 86 86 162 162 162 F F F
108622024-01-10T17:59:34.829ZMax gen: 100, Max flush: 193
108632024-01-10T17:59:34.829ZError: Difference in extent metadata found!
108642024-01-10T17:59:34.832ZA Difference in extent metadata is expected here
108652024-01-10T17:59:34.834ZOn loop 50
108662024-01-10T17:59:34.834Z
108672024-01-10T17:59:34.837ZVerifying data now
108682024-01-10T17:59:36.370ZNo change after verify
108692024-01-10T17:59:36.371ZLoop: 50 Downstairs dump after verify (and repair):
108702024-01-10T17:59:36.394ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108712024-01-10T17:59:36.394Z 0 000-019 98 98 98 187 187 187 F F F
108722024-01-10T17:59:36.394Z 1 020-039 101 101 101 194 194 194 F F F
108732024-01-10T17:59:36.394Z 2 040-059 100 100 100 190 190 190 F F F
108742024-01-10T17:59:36.394Z 3 060-079 97 97 97 186 186 186 F F F
108752024-01-10T17:59:36.394Z 4 080-099 99 99 99 189 189 189 F F F
108762024-01-10T17:59:36.394Z 5 100-119 96 96 96 183 183 183 F F F
108772024-01-10T17:59:36.394Z 6 120-139 100 100 100 193 193 193 F F F
108782024-01-10T17:59:36.399Z 7 140-159 100 100 100 193 193 193 F F F
108792024-01-10T17:59:36.399Z 8 160-179 97 97 97 186 186 186 F F F
108802024-01-10T17:59:36.399Z 9 180-199 97 97 97 186 186 186 F F F
108812024-01-10T17:59:36.399Z 10 200-219 100 100 100 193 193 193 F F F
108822024-01-10T17:59:36.399Z 11 220-239 100 100 100 191 191 191 F F F
108832024-01-10T17:59:36.399Z 12 240-259 100 100 100 191 191 191 F F F
108842024-01-10T17:59:36.399Z 13 260-279 100 100 100 193 193 193 F F F
108852024-01-10T17:59:36.402Z 14 280-299 98 98 98 187 187 187 F F F
108862024-01-10T17:59:36.402Z 15 300-319 99 99 99 189 189 189 F F F
108872024-01-10T17:59:36.403Z 16 320-339 99 99 99 189 189 189 F F F
108882024-01-10T17:59:36.403Z 17 340-359 101 101 101 194 194 194 F F F
108892024-01-10T17:59:36.403Z 18 360-379 99 99 99 189 189 189 F F F
108902024-01-10T17:59:36.403Z 19 380-399 96 96 96 184 184 184 F F F
108912024-01-10T17:59:36.403Z 20 400-419 100 100 100 190 190 190 F F F
108922024-01-10T17:59:36.403Z 21 420-439 100 100 100 190 190 190 F F F
108932024-01-10T17:59:36.403Z 22 440-459 92 92 92 176 176 176 F F F
108942024-01-10T17:59:36.403Z 23 460-479 100 100 100 190 190 190 F F F
108952024-01-10T17:59:36.403Z 24 480-499 100 100 100 193 193 193 F F F
108962024-01-10T17:59:36.403Z 25 500-519 100 100 100 190 190 190 F F F
108972024-01-10T17:59:36.403Z 26 520-539 100 100 100 191 191 191 F F F
108982024-01-10T17:59:36.403Z 27 540-559 100 100 100 190 190 190 F F F
108992024-01-10T17:59:36.403Z 28 560-579 99 99 99 189 189 189 F F F
109002024-01-10T17:59:36.403Z 29 580-599 86 86 86 162 162 162 F F F
109012024-01-10T17:59:36.403ZMax gen: 101, Max flush: 194
109022024-01-10T17:59:36.403Z
109032024-01-10T17:59:36.404ZBegin loop 51
109042024-01-10T17:59:36.404ZDownstairs to restart: 0
109052024-01-10T17:59:36.432Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.435964023Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563}
109062024-01-10T17:59:36.432Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.436501527Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563}
109072024-01-10T17:59:36.436Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.436596068Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563}
109082024-01-10T17:59:36.436Z{"msg":"Crucible 30467064-b2fc-419b-9a7f-6d86c2524902 has session id: 11d8ec64-da1b-435a-8aa3-86162ed2081f","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.436637298Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109092024-01-10T17:59:36.436Z{"msg":"Upstairs opts: Upstairs UUID: 30467064-b2fc-419b-9a7f-6d86c2524902, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.436769905Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109102024-01-10T17:59:36.436Z{"msg":"Crucible stats registered with UUID: 30467064-b2fc-419b-9a7f-6d86c2524902","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.436848708Z","Crucible runtime is spawned
109112024-01-10T17:59:36.436Zhostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109122024-01-10T17:59:36.436Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.437040718Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563}
109132024-01-10T17:59:36.436Z{"msg":"30467064-b2fc-419b-9a7f-6d86c2524902 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.437260289Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109142024-01-10T17:59:36.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.437297054Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"0","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109152024-01-10T17:59:36.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.437321294Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"0","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109162024-01-10T17:59:36.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.437393079Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"1","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109172024-01-10T17:59:36.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.437508151Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"1","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109182024-01-10T17:59:36.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.437638233Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"2","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109192024-01-10T17:59:36.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.437823521Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"2","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109202024-01-10T17:59:36.437Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.43787686Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109212024-01-10T17:59:36.437Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.438000082Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109222024-01-10T17:59:36.437Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.438031052Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109232024-01-10T17:59:36.437Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.438054039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109242024-01-10T17:59:36.437Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.438092363Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109252024-01-10T17:59:36.439Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.438151969Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109262024-01-10T17:59:36.439Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.438937275Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"1","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109272024-01-10T17:59:36.439Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.439012529Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"1","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109282024-01-10T17:59:36.439Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.439106Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"2","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109292024-01-10T17:59:36.440Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.439190886Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"2","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109302024-01-10T17:59:36.440Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.439933102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"2","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109312024-01-10T17:59:36.444Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.440227378Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"1","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109322024-01-10T17:59:36.444Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.44082879Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"2","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109332024-01-10T17:59:36.444Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.440925475Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"2","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109342024-01-10T17:59:36.444Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.440966723Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109352024-01-10T17:59:36.445Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.441008933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"1","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109362024-01-10T17:59:36.445Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.441078517Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"1","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109372024-01-10T17:59:36.445Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.441106666Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109382024-01-10T17:59:36.445Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.441144614Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"0","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109392024-01-10T17:59:36.445Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.441204776Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"0","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109402024-01-10T17:59:36.445Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.442030711Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"0","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109412024-01-10T17:59:36.445Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.442731746Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"0","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109422024-01-10T17:59:36.445Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.442835671Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"0","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109432024-01-10T17:59:36.446Z{"msg":"[0]R flush_numbers[0..12]: [187, 194, 190, 186, 189, 183, 193, 193, 186, 186, 193, 191]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.442910206Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109442024-01-10T17:59:36.446Z{"msg":"[0]R generation[0..12]: [98, 101, 100, 97, 99, 96, 100, 100, 97, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.442981041Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109452024-01-10T17:59:36.446Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.443054601Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109462024-01-10T17:59:36.446Z{"msg":"[1]R flush_numbers[0..12]: [187, 194, 190, 186, 189, 183, 193, 193, 186, 186, 193, 191]","v":Wait for a query_work_queue command to finish before sending IO
109472024-01-10T17:59:36.446Z0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.443221518Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109482024-01-10T17:59:36.446Z{"msg":"[1]R generation[0..12]: [98, 101, 100, 97, 99, 96, 100, 100, 97, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.443292995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109492024-01-10T17:59:36.446Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.443366796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109502024-01-10T17:59:36.446Z{"msg":"[2]R flush_numbers[0..12]: [187, 194, 190, 186, 189, 183, 193, 193, 186, 186, 193, 191]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.443443499Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109512024-01-10T17:59:36.447Z{"msg":"[2]R generation[0..12]: [98, 101, 100, 97, 99, 96, 100, 100, 97, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.443521458Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109522024-01-10T17:59:36.447Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.443600379Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109532024-01-10T17:59:36.447Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.443675607Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109542024-01-10T17:59:36.447Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.443722949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109552024-01-10T17:59:36.447Z{"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.443793687Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109562024-01-10T17:59:36.447Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.44386392Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"":"downstairs","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109572024-01-10T17:59:36.447Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
109582024-01-10T17:59:36.447Z,"time":"2024-01-10T17:59:36.443933054Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109592024-01-10T17:59:36.447Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.444004252Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"0","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109602024-01-10T17:59:36.447Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.444055784Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"1","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109612024-01-10T17:59:36.447Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.444079821Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"client":"2","":"downstairs","session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109622024-01-10T17:59:36.448Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.444119563Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109632024-01-10T17:59:36.448Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.444141116Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109642024-01-10T17:59:36.448Z{"msg":"30467064-b2fc-419b-9a7f-6d86c2524902 is now active with session: 11d8ec64-da1b-435a-8aa3-86162ed2081f","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.444163632Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109652024-01-10T17:59:36.448Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.444200561Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563,"session_id":"11d8ec64-da1b-435a-8aa3-86162ed2081f"}
109662024-01-10T17:59:36.448Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:36.444221829Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1563}
109672024-01-10T17:59:36.448ZLoading write count information from file "/tmp/test_repair_verify.data"
109682024-01-10T17:59:36.448ZRun Repair workload
109692024-01-10T17:59:36.448Z01/30 Write block 234 len 24576 data: 3 3 5 6 6 7
109702024-01-10T17:59:36.449Z02/30 Flush
109712024-01-10T17:59:36.452Z03/30 Write block 149 len 32768 data: 9 10 9 7 7 7 7 6
109722024-01-10T17:59:36.452Z04/30 Write block 221 len 8192 data: 7 6
109732024-01-10T17:59:36.452Z05/30 Write block 109 len 36864 data: 8 10 11 13 13 10 8 8 8
109742024-01-10T17:59:36.452Z06/30 Write block 34 len 32768 data: 11 10 10 9 9 6 5 6
109752024-01-10T17:59:36.452Z07/30 Flush
109762024-01-10T17:59:36.457Z08/30 Flush
109772024-01-10T17:59:36.460Z09/30 Write block 32 len 16384 data: 13 12 12 11
109782024-01-10T17:59:36.460Z10/30 Write block 530 len 8192 data: 13 16
109792024-01-10T17:59:36.460Z11/30 Write block 88 len 28672 data: 5 6 5 8 7 7 8
109802024-01-10T17:59:36.464Z12/30 Read block 235 len 40960
109812024-01-10T17:59:36.467Z13/30 Write block 172 len 16384 data: 7 7 8 7
109822024-01-10T17:59:36.467Z14/30 Read block 387 len 40960
109832024-01-10T17:59:36.470Z15/30 Read block 495 len 4096
109842024-01-10T17:59:36.473Z16/30 Read block 327 len 36864
109852024-01-10T17:59:36.476Z17/30 Read block 400 len 40960
109862024-01-10T17:59:36.479Z18/30 Read block 331 len 32768
109872024-01-10T17:59:36.482Z19/30 Read block 336 len 16384
109882024-01-10T17:59:36.485Z20/30 Write block 75 len 16384 data: 8 9 8 6
109892024-01-10T17:59:36.485Z21/30 Read block 390 len 28672
109902024-01-10T17:59:36.489Z22/30 Read block 555 len 12288
109912024-01-10T17:59:36.489Z23/30 Read block 436 len 36864
109922024-01-10T17:59:36.494Z24/30 Flush
109932024-01-10T17:59:36.498Z25/30 Write block 221 len 8192 data: 8 7
109942024-01-10T17:59:36.498Z26/30 Read block 537 len 40960
109952024-01-10T17:59:36.501Z27/30 Read block 515 len 4096
109962024-01-10T17:59:36.501Z28/30 Read block 580 len 8192
109972024-01-10T17:59:36.501Z29/30 Read block 259 len 36864
109982024-01-10T17:59:36.504Z30/30 Write block 141 len 36864 data: 11 9 9 9 8 8 10 10 10
109992024-01-10T17:59:36.508Z----------------------------------------------------------------
110002024-01-10T17:59:36.508Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:22
110012024-01-10T17:59:36.508ZStates: Active Active Active
110022024-01-10T17:59:36.508ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110032024-01-10T17:59:36.508Z 9 Acked 1008 Write 4 Sent Done Done false
110042024-01-10T17:59:36.508Z 10 Acked 1009 Write 2 Sent Done Done false
110052024-01-10T17:59:36.508Z 11 Acked 1010 Write 7 Sent Done Done false
110062024-01-10T17:59:36.508Z 12 Acked 1011 Read 10 Sent Done Done false
110072024-01-10T17:59:36.508Z 13 Acked 1012 Write 4 Sent Done Done false
110082024-01-10T17:59:36.508Z 14 Acked 1013 Read 10 Sent Done Done false
110092024-01-10T17:59:36.508Z 15 Acked 1014 Read 1 Sent Done Done false
110102024-01-10T17:59:36.508Z 16 Acked 1015 Read 9 Sent Done Done false
110112024-01-10T17:59:36.508Z 17 Acked 1016 Read 10 Sent Done Done false
110122024-01-10T17:59:36.508Z 18 Acked 1017 Read 8 Sent Done Done false
110132024-01-10T17:59:36.508Z 19 Acked 1018 Read 4 Sent Done Done false
110142024-01-10T17:59:36.508Z 20 Acked 1019 Write 4 Sent Done Done false
110152024-01-10T17:59:36.509Z 21 Acked 1020 Read 7 Sent Done Done false
110162024-01-10T17:59:36.509Z 22 Acked 1021 Read 3 Sent Done Done false
110172024-01-10T17:59:36.509Z 23 Acked 1022 Read 9 Sent Done Done false
110182024-01-10T17:59:36.509Z 24 Acked 1023 Flush 0 Sent Done Done false
110192024-01-10T17:59:36.509Z 25 Acked 1024 Write 2 Sent Done Done false
110202024-01-10T17:59:36.509Z 26 Acked 1025 Read 10 Sent Done Done false
110212024-01-10T17:59:36.509Z 27 Acked 1026 Read 1 Sent Done Done false
110222024-01-10T17:59:36.509Z 28 Acked 1027 Read 2 Sent Done Done false
110232024-01-10T17:59:36.510Z 29 Acked 1028 Read 9 Sent Done Done false
110242024-01-10T17:59:36.510Z 30 Acked 1029 Write 9 Sent Sent Sent false
110252024-01-10T17:59:36.510Z STATES DS:0 DS:1 DS:2 TOTAL
110262024-01-10T17:59:36.510Z New 0 0 0 0
110272024-01-10T17:59:36.510Z Sent 22 1 1 24
110282024-01-10T17:59:36.510Z Done 0 21 21 42
110292024-01-10T17:59:36.510Z Skipped 0 0 0 0
110302024-01-10T17:59:36.510Z Error 0 0 0 0
110312024-01-10T17:59:36.510ZLast Flush: 1007 1023 1023
110322024-01-10T17:59:36.510ZDownstairs last five completed: 1007 1006 1005 1004 1003
110332024-01-10T17:59:36.510ZUpstairs last five completed: 30 29 28 27 26
110342024-01-10T17:59:36.511ZWrote out file "/tmp/test_repair_verify.data"
110352024-01-10T17:59:36.514Z
110362024-01-10T17:59:36.519ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
110372024-01-10T17:59:36.543ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110382024-01-10T17:59:36.543Z 0 000-019 98 98 98 187 187 187 F F F
110392024-01-10T17:59:36.544Z 1 020-039 102 102 102 196 198 198 F F F <---
110402024-01-10T17:59:36.544Z 2 040-059 102 102 102 196 196 196 F F F
110412024-01-10T17:59:36.544Z 3 060-079 97 102 102 186 198 198 F F F <---
110422024-01-10T17:59:36.544Z 4 080-099 99 102 102 189 198 198 F F F <---
110432024-01-10T17:59:36.544Z 5 100-119 102 102 102 196 196 196 F F F
110442024-01-10T17:59:36.544Z 6 120-139 100 100 100 193 193 193 F F F
110452024-01-10T17:59:36.547Z 7 140-159 102 102 102 196 196 196 F T T <---
110462024-01-10T17:59:36.547Z 8 160-179 97 102 102 186 198 198 F F F <---
110472024-01-10T17:59:36.547Z 9 180-199 97 97 97 186 186 186 F F F
110482024-01-10T17:59:36.548Z 10 200-219 100 100 100 193 193 193 F F F
110492024-01-10T17:59:36.548Z 11 220-239 102 102 102 196 196 196 F T T <---
110502024-01-10T17:59:36.548Z 12 240-259 100 100 100 191 191 191 F F F
110512024-01-10T17:59:36.548Z 13 260-279 100 100 100 193 193 193 F F F
110522024-01-10T17:59:36.548Z 14 280-299 98 98 98 187 187 187 F F F
110532024-01-10T17:59:36.548Z 15 300-319 99 99 99 189 189 189 F F F
110542024-01-10T17:59:36.548Z 16 320-339 99 99 99 189 189 189 F F F
110552024-01-10T17:59:36.548Z 17 340-359 101 101 101 194 194 194 F F F
110562024-01-10T17:59:36.548Z 18 360-379 99 99 99 189 189 189 F F F
110572024-01-10T17:59:36.548Z 19 380-399 96 96 96 184 184 184 F F F
110582024-01-10T17:59:36.548Z 20 400-419 100 100 100 190 190 190 F F F
110592024-01-10T17:59:36.548Z 21 420-439 100 100 100 190 190 190 F F F
110602024-01-10T17:59:36.548Z 22 440-459 92 92 92 176 176 176 F F F
110612024-01-10T17:59:36.548Z 23 460-479 100 100 100 190 190 190 F F F
110622024-01-10T17:59:36.548Z 24 480-499 100 100 100 193 193 193 F F F
110632024-01-10T17:59:36.549Z 25 500-519 100 100 100 190 190 190 F F F
110642024-01-10T17:59:36.549Z 26 520-539 100 102 102 191 198 198 F F F <---
110652024-01-10T17:59:36.549Z 27 540-559 100 100 100 190 190 190 F F F
110662024-01-10T17:59:36.549Z 28 560-579 99 99 99 189 189 189 F F F
110672024-01-10T17:59:36.549Z 29 580-599 86 86 86 162 162 162 F F F
110682024-01-10T17:59:36.549ZMax gen: 102, Max flush: 198
110692024-01-10T17:59:36.549ZError: Difference in extent metadata found!
110702024-01-10T17:59:36.586ZA Difference in extent metadata is expected here
110712024-01-10T17:59:36.586ZOn loop 51
110722024-01-10T17:59:36.586Z
110732024-01-10T17:59:36.586ZVerifying data now
110742024-01-10T17:59:41.384ZNo change after verify
110752024-01-10T17:59:41.384ZLoop: 51 Downstairs dump after verify (and repair):
110762024-01-10T17:59:41.400ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110772024-01-10T17:59:41.400Z 0 000-019 98 98 98 187 187 187 F F F
110782024-01-10T17:59:41.400Z 1 020-039 102 102 102 198 198 198 F F F
110792024-01-10T17:59:41.400Z 2 040-059 102 102 102 196 196 196 F F F
110802024-01-10T17:59:41.400Z 3 060-079 102 102 102 198 198 198 F F F
110812024-01-10T17:59:41.400Z 4 080-099 102 102 102 198 198 198 F F F
110822024-01-10T17:59:41.400Z 5 100-119 102 102 102 196 196 196 F F F
110832024-01-10T17:59:41.400Z 6 120-139 100 100 100 193 193 193 F F F
110842024-01-10T17:59:41.400Z 7 140-159 103 103 103 199 199 199 F F F
110852024-01-10T17:59:41.400Z 8 160-179 102 102 102 198 198 198 F F F
110862024-01-10T17:59:41.403Z 9 180-199 97 97 97 186 186 186 F F F
110872024-01-10T17:59:41.403Z 10 200-219 100 100 100 193 193 193 F F F
110882024-01-10T17:59:41.403Z 11 220-239 103 103 103 199 199 199 F F F
110892024-01-10T17:59:41.403Z 12 240-259 100 100 100 191 191 191 F F F
110902024-01-10T17:59:41.404Z 13 260-279 100 100 100 193 193 193 F F F
110912024-01-10T17:59:41.405Z 14 280-299 98 98 98 187 187 187 F F F
110922024-01-10T17:59:41.405Z 15 300-319 99 99 99 189 189 189 F F F
110932024-01-10T17:59:41.405Z 16 320-339 99 99 99 189 189 189 F F F
110942024-01-10T17:59:41.405Z 17 340-359 101 101 101 194 194 194 F F F
110952024-01-10T17:59:41.405Z 18 360-379 99 99 99 189 189 189 F F F
110962024-01-10T17:59:41.405Z 19 380-399 96 96 96 184 184 184 F F F
110972024-01-10T17:59:41.405Z 20 400-419 100 100 100 190 190 190 F F F
110982024-01-10T17:59:41.405Z 21 420-439 100 100 100 190 190 190 F F F
110992024-01-10T17:59:41.405Z 22 440-459 92 92 92 176 176 176 F F F
111002024-01-10T17:59:41.405Z 23 460-479 100 100 100 190 190 190 F F F
111012024-01-10T17:59:41.405Z 24 480-499 100 100 100 193 193 193 F F F
111022024-01-10T17:59:41.405Z 25 500-519 100 100 100 190 190 190 F F F
111032024-01-10T17:59:41.405Z 26 520-539 102 102 102 198 198 198 F F F
111042024-01-10T17:59:41.405Z 27 540-559 100 100 100 190 190 190 F F F
111052024-01-10T17:59:41.405Z 28 560-579 99 99 99 189 189 189 F F F
111062024-01-10T17:59:41.405Z 29 580-599 86 86 86 162 162 162 F F F
111072024-01-10T17:59:41.405ZMax gen: 103, Max flush: 199
111082024-01-10T17:59:41.410Z
111092024-01-10T17:59:41.410ZBegin loop 52
111102024-01-10T17:59:41.410ZDownstairs to restart: 1
111112024-01-10T17:59:41.435Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.438991949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571}
111122024-01-10T17:59:41.439Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.439482807Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571}
111132024-01-10T17:59:41.440Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.439568208Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571}
111142024-01-10T17:59:41.440Z{"msg":"Crucible 5f8f4795-5847-4bcb-ba01-1901e9cc2c63 has session id: 152c09ce-eec1-45d1-b865-b57e020d1cbf","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.439692573Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111152024-01-10T17:59:41.440Z{"msg":"Upstairs opts: Upstairs UUID: 5f8f4795-5847-4bcb-ba01-1901e9cc2c63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
111162024-01-10T17:59:41.440Z,"time":"2024-01-10T17:59:41.439819314Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111172024-01-10T17:59:41.440Z{"msg":"Crucible stats registered with UUID: 5f8f4795-5847-4bcb-ba01-1901e9cc2c63","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.439949041Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111182024-01-10T17:59:41.440Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440006716Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571}
111192024-01-10T17:59:41.440Z{"msg":"5f8f4795-5847-4bcb-ba01-1901e9cc2c63 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440211863Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111202024-01-10T17:59:41.440Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440274683Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"0","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111212024-01-10T17:59:41.440Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440374813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"0","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111222024-01-10T17:59:41.440Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440449465Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"1","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111232024-01-10T17:59:41.440Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440522787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"1","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111242024-01-10T17:59:41.441Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440611172Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"2","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111252024-01-10T17:59:41.441Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440660857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"2","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111262024-01-10T17:59:41.441Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440704703Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111272024-01-10T17:59:41.441Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440749995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111282024-01-10T17:59:41.441Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440793699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111292024-01-10T17:59:41.441Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440825647Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111302024-01-10T17:59:41.441Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440860109Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111312024-01-10T17:59:41.441Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.440948764Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111322024-01-10T17:59:41.441Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.441933912Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"2","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111332024-01-10T17:59:41.442Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.442053568Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"2","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111342024-01-10T17:59:41.442Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.442102091Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"0","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111352024-01-10T17:59:41.442Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.442178934Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"0","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111362024-01-10T17:59:41.446Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.442907055Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"0","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111372024-01-10T17:59:41.446Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.44318604Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"2","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111382024-01-10T17:59:41.446Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.443277018Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"1","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111392024-01-10T17:59:41.447Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.443323861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"1","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111402024-01-10T17:59:41.447Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.443754304Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"0","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111412024-01-10T17:59:41.447Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.443824707Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"0","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111422024-01-10T17:59:41.447Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.443870388Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111432024-01-10T17:59:41.447Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.44419527Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"2","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111442024-01-10T17:59:41.447Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.444280435Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"2","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111452024-01-10T17:59:41.447Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.44436207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111462024-01-10T17:59:41.447Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.444439303Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"1","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111472024-01-10T17:59:41.447Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445063847Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"1","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111482024-01-10T17:59:41.447Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445147164Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"1","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111492024-01-10T17:59:41.447Z{"msg":"[0]R flush_numbers[0..12]: [187, 198, 196, 198, 198, 196, 193, 199, 198, 186, 193, 199]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445222647Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111502024-01-10T17:59:41.447Z{"msg":"[0]R generation[0..12]: [98, 102, 102, 102, 102, 102, 100, 103, 102, 97, 100, 103]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.44529497Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111512024-01-10T17:59:41.447Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445370099Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111522024-01-10T17:59:41.447Z{"msg":"[1]R flush_numbers[0..12]: [187, 198, 196, 198, 198, 196, 193, 199, 198, 186, 193, 199]"Wait for a query_work_queue command to finish before sending IO
111532024-01-10T17:59:41.448Z,"v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.44553033Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111542024-01-10T17:59:41.448Z{"msg":"[1]R generation[0..12]: [98, 102, 102, 102, 102, 102, 100, 103, 102, 97, 100, 103]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445609801Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111552024-01-10T17:59:41.448Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445710098Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111562024-01-10T17:59:41.448Z{"msg":"[2]R flush_numbers[0..12]: [187, 198, 196, 198, 198, 196, 193, 199, 198, 186, 193, 199]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445783154Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111572024-01-10T17:59:41.448Z{"msg":"[2]R generation[0..12]: [98, 102, 102, 102, 102, 102, 100, 103, 102, 97, 100, 103]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445828388Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111582024-01-10T17:59:41.448Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445877553Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111592024-01-10T17:59:41.448Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445920437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111602024-01-10T17:59:41.448Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.445988953Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111612024-01-10T17:59:41.448Z{"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446058717Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111622024-01-10T17:59:41.448Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446087829Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"":"downstairs","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111632024-01-10T17:59:41.449Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446151938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111642024-01-10T17:59:41.449Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446221359Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"0","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111652024-01-10T17:59:41.449Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446295525Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"1","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111662024-01-10T17:59:41.450Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446342727Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"client":"2","":"downstairs","session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111672024-01-10T17:59:41.450Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446414276Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
111682024-01-10T17:59:41.450Z,"pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111692024-01-10T17:59:41.450Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446534055Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111702024-01-10T17:59:41.450Z{"msg":"5f8f4795-5847-4bcb-ba01-1901e9cc2c63 is now active with session: 152c09ce-eec1-45d1-b865-b57e020d1cbf","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446583869Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111712024-01-10T17:59:41.450Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446605343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571,"session_id":"152c09ce-eec1-45d1-b865-b57e020d1cbf"}
111722024-01-10T17:59:41.450Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:41.446643636Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1571}
111732024-01-10T17:59:41.450ZLoading write count information from file "/tmp/test_repair_verify.data"
111742024-01-10T17:59:41.450ZRun Repair workload
111752024-01-10T17:59:41.450Z01/30 Write block 207 len 12288 data: 4 4 6
111762024-01-10T17:59:41.453Z02/30 Write block 84 len 16384 data: 6 5 5 6
111772024-01-10T17:59:41.453Z03/30 Read block 423 len 32768
111782024-01-10T17:59:41.453Z04/30 Write block 155 len 16384 data: 8 7 5 3
111792024-01-10T17:59:41.453Z05/30 Write block 90 len 4096 data: 6
111802024-01-10T17:59:41.453Z06/30 Write block 297 len 12288 data: 11 13 11
111812024-01-10T17:59:41.453Z07/30 Read block 45 len 36864
111822024-01-10T17:59:41.456Z08/30 Read block 479 len 12288
111832024-01-10T17:59:41.459Z09/30 Read block 173 len 16384
111842024-01-10T17:59:41.462Z10/30 Write block 228 len 32768 data: 7 7 6 4 2 3 4 4
111852024-01-10T17:59:41.462Z11/30 Read block 15 len 36864
111862024-01-10T17:59:41.465Z12/30 Read block 239 len 36864
111872024-01-10T17:59:41.468Z13/30 Read block 154 len 4096
111882024-01-10T17:59:41.470Z14/30 Read block 399 len 24576
111892024-01-10T17:59:41.471Z15/30 Read block 533 len 12288
111902024-01-10T17:59:41.474Z16/30 Read block 347 len 36864
111912024-01-10T17:59:41.477Z17/30 Read block 239 len 20480
111922024-01-10T17:59:41.480Z18/30 Read block 68 len 4096
111932024-01-10T17:59:41.480Z19/30 Write block 156 len 16384 data: 8 6 4 3
111942024-01-10T17:59:41.480Z20/30 Read block 248 len 32768
111952024-01-10T17:59:41.482Z21/30 Read block 437 len 24576
111962024-01-10T17:59:41.485Z22/30 Read block 350 len 8192
111972024-01-10T17:59:41.488Z23/30 Read block 418 len 20480
111982024-01-10T17:59:41.488Z24/30 Read block 150 len 40960
111992024-01-10T17:59:41.492Z25/30 Write block 556 len 36864 data: 10 9 7 7 8 9 9 9 9
112002024-01-10T17:59:41.494Z26/30 Read block 44 len 32768
112012024-01-10T17:59:41.497Z27/30 Read block 324 len 12288
112022024-01-10T17:59:41.499Z28/30 Read block 302 len 28672
112032024-01-10T17:59:41.502Z29/30 Read block 586 len 12288
112042024-01-10T17:59:41.502Z30/30 Write block 228 len 20480 data: 8 8 7 5 3
112052024-01-10T17:59:41.502Z----------------------------------------------------------------
112062024-01-10T17:59:41.502Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
112072024-01-10T17:59:41.507ZStates: Active Active Active
112082024-01-10T17:59:41.507ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112092024-01-10T17:59:41.507Z 1 Acked 1000 Write 3 Done Sent Done false
112102024-01-10T17:59:41.507Z 2 Acked 1001 Write 4 Done Sent Done false
112112024-01-10T17:59:41.507Z 3 Acked 1002 Read 8 Done Sent Done false
112122024-01-10T17:59:41.507Z 4 Acked 1003 Write 4 Done Sent Done false
112132024-01-10T17:59:41.507Z 5 Acked 1004 Write 1 Done Sent Done false
112142024-01-10T17:59:41.507Z 6 Acked 1005 Write 3 Done Sent Done false
112152024-01-10T17:59:41.507Z 7 Acked 1006 Read 9 Done Sent Done false
112162024-01-10T17:59:41.507Z 8 Acked 1007 Read 3 Done Sent Done false
112172024-01-10T17:59:41.507Z 9 Acked 1008 Read 4 Done Sent Done false
112182024-01-10T17:59:41.507Z 10 Acked 1009 Write 8 Done Sent Done false
112192024-01-10T17:59:41.507Z 11 Acked 1010 Read 9 Done Sent Done false
112202024-01-10T17:59:41.507Z 12 Acked 1011 Read 9 Done Sent Done false
112212024-01-10T17:59:41.507Z 13 Acked 1012 Read 1 Done Sent Done false
112222024-01-10T17:59:41.507Z 14 Acked 1013 Read 6 Done Sent Done false
112232024-01-10T17:59:41.507Z 15 Acked 1014 Read 3 Done Sent Done false
112242024-01-10T17:59:41.507Z 16 Acked 1015 Read 9 Done Sent Done false
112252024-01-10T17:59:41.507Z 17 Acked 1016 Read 5 Done Sent Done false
112262024-01-10T17:59:41.508Z 18 Acked 1017 Read 1 Done Sent Done false
112272024-01-10T17:59:41.508Z 19 Acked 1018 Write 4 Done Sent Done false
112282024-01-10T17:59:41.509Z 20 Acked 1019 Read 8 Done Sent Done false
112292024-01-10T17:59:41.509Z 21 Acked 1020 Read 6 Done Sent Done false
112302024-01-10T17:59:41.509Z 22 Acked 1021 Read 2 Done Sent Done false
112312024-01-10T17:59:41.509Z 23 Acked 1022 Read 5 Done Sent Done false
112322024-01-10T17:59:41.509Z 24 Acked 1023 Read 10 Done Sent Done false
112332024-01-10T17:59:41.509Z 25 Acked 1024 Write 9 Done Sent Done false
112342024-01-10T17:59:41.509Z 26 Acked 1025 Read 8 Done Sent Done false
112352024-01-10T17:59:41.509Z 27 Acked 1026 Read 3 Done Sent Done false
112362024-01-10T17:59:41.509Z 28 Acked 1027 Read 7 Done Sent Done false
112372024-01-10T17:59:41.509Z 29 Acked 1028 Read 3 Done Sent Done false
112382024-01-10T17:59:41.509Z 30 Acked 1029 Write 5 Sent Sent Sent false
112392024-01-10T17:59:41.509Z STATES DS:0 DS:1 DS:2 TOTAL
112402024-01-10T17:59:41.509Z New 0 0 0 0
112412024-01-10T17:59:41.509Z Sent 1 30 1 32
112422024-01-10T17:59:41.509Z Done 29 0 29 58
112432024-01-10T17:59:41.509Z Skipped 0 0 0 0
112442024-01-10T17:59:41.512Z Error 0 0 0 0
112452024-01-10T17:59:41.512ZLast Flush: 0 0 0
112462024-01-10T17:59:41.512ZDownstairs last five completed:
112472024-01-10T17:59:41.512ZUpstairs last five completed: 30 29 28 27 26
112482024-01-10T17:59:41.512ZWrote out file "/tmp/test_repair_verify.data"
112492024-01-10T17:59:41.516Z
112502024-01-10T17:59:41.519ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
112512024-01-10T17:59:41.542ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112522024-01-10T17:59:41.542Z 0 000-019 98 98 98 187 187 187 F F F
112532024-01-10T17:59:41.542Z 1 020-039 102 102 102 198 198 198 F F F
112542024-01-10T17:59:41.542Z 2 040-059 102 102 102 196 196 196 F F F
112552024-01-10T17:59:41.542Z 3 060-079 102 102 102 198 198 198 F F F
112562024-01-10T17:59:41.542Z 4 080-099 102 102 102 198 198 198 T F T <---
112572024-01-10T17:59:41.542Z 5 100-119 102 102 102 196 196 196 F F F
112582024-01-10T17:59:41.542Z 6 120-139 100 100 100 193 193 193 F F F
112592024-01-10T17:59:41.546Z 7 140-159 103 103 103 199 199 199 T F T <---
112602024-01-10T17:59:41.546Z 8 160-179 102 102 102 198 198 198 F F F
112612024-01-10T17:59:41.546Z 9 180-199 97 97 97 186 186 186 F F F
112622024-01-10T17:59:41.546Z 10 200-219 100 100 100 193 193 193 T F T <---
112632024-01-10T17:59:41.546Z 11 220-239 103 103 103 199 199 199 T F T <---
112642024-01-10T17:59:41.546Z 12 240-259 100 100 100 191 191 191 F F F
112652024-01-10T17:59:41.546Z 13 260-279 100 100 100 193 193 193 F F F
112662024-01-10T17:59:41.546Z 14 280-299 98 98 98 187 187 187 T F T <---
112672024-01-10T17:59:41.546Z 15 300-319 99 99 99 189 189 189 F F F
112682024-01-10T17:59:41.546Z 16 320-339 99 99 99 189 189 189 F F F
112692024-01-10T17:59:41.546Z 17 340-359 101 101 101 194 194 194 F F F
112702024-01-10T17:59:41.546Z 18 360-379 99 99 99 189 189 189 F F F
112712024-01-10T17:59:41.546Z 19 380-399 96 96 96 184 184 184 F F F
112722024-01-10T17:59:41.546Z 20 400-419 100 100 100 190 190 190 F F F
112732024-01-10T17:59:41.546Z 21 420-439 100 100 100 190 190 190 F F F
112742024-01-10T17:59:41.546Z 22 440-459 92 92 92 176 176 176 F F F
112752024-01-10T17:59:41.547Z 23 460-479 100 100 100 190 190 190 F F F
112762024-01-10T17:59:41.547Z 24 480-499 100 100 100 193 193 193 F F F
112772024-01-10T17:59:41.548Z 25 500-519 100 100 100 190 190 190 F F F
112782024-01-10T17:59:41.548Z 26 520-539 102 102 102 198 198 198 F F F
112792024-01-10T17:59:41.548Z 27 540-559 100 100 100 190 190 190 T F T <---
112802024-01-10T17:59:41.548Z 28 560-579 99 99 99 189 189 189 T F T <---
112812024-01-10T17:59:41.548Z 29 580-599 86 86 86 162 162 162 F F F
112822024-01-10T17:59:41.548ZMax gen: 103, Max flush: 199
112832024-01-10T17:59:41.548ZError: Difference in extent metadata found!
112842024-01-10T17:59:41.551ZA Difference in extent metadata is expected here
112852024-01-10T17:59:41.551ZOn loop 52
112862024-01-10T17:59:41.551Z
112872024-01-10T17:59:41.551ZVerifying data now
112882024-01-10T17:59:47.391ZNo change after verify
112892024-01-10T17:59:47.391ZLoop: 52 Downstairs dump after verify (and repair):
112902024-01-10T17:59:47.414ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112912024-01-10T17:59:47.414Z 0 000-019 98 98 98 187 187 187 F F F
112922024-01-10T17:59:47.414Z 1 020-039 102 102 102 198 198 198 F F F
112932024-01-10T17:59:47.414Z 2 040-059 102 102 102 196 196 196 F F F
112942024-01-10T17:59:47.414Z 3 060-079 102 102 102 198 198 198 F F F
112952024-01-10T17:59:47.414Z 4 080-099 104 104 104 200 200 200 F F F
112962024-01-10T17:59:47.415Z 5 100-119 102 102 102 196 196 196 F F F
112972024-01-10T17:59:47.415Z 6 120-139 100 100 100 193 193 193 F F F
112982024-01-10T17:59:47.418Z 7 140-159 104 104 104 200 200 200 F F F
112992024-01-10T17:59:47.418Z 8 160-179 102 102 102 198 198 198 F F F
113002024-01-10T17:59:47.418Z 9 180-199 97 97 97 186 186 186 F F F
113012024-01-10T17:59:47.418Z 10 200-219 104 104 104 200 200 200 F F F
113022024-01-10T17:59:47.418Z 11 220-239 104 104 104 200 200 200 F F F
113032024-01-10T17:59:47.418Z 12 240-259 100 100 100 191 191 191 F F F
113042024-01-10T17:59:47.419Z 13 260-279 100 100 100 193 193 193 F F F
113052024-01-10T17:59:47.419Z 14 280-299 104 104 104 200 200 200 F F F
113062024-01-10T17:59:47.419Z 15 300-319 99 99 99 189 189 189 F F F
113072024-01-10T17:59:47.419Z 16 320-339 99 99 99 189 189 189 F F F
113082024-01-10T17:59:47.419Z 17 340-359 101 101 101 194 194 194 F F F
113092024-01-10T17:59:47.419Z 18 360-379 99 99 99 189 189 189 F F F
113102024-01-10T17:59:47.419Z 19 380-399 96 96 96 184 184 184 F F F
113112024-01-10T17:59:47.419Z 20 400-419 100 100 100 190 190 190 F F F
113122024-01-10T17:59:47.419Z 21 420-439 100 100 100 190 190 190 F F F
113132024-01-10T17:59:47.419Z 22 440-459 92 92 92 176 176 176 F F F
113142024-01-10T17:59:47.419Z 23 460-479 100 100 100 190 190 190 F F F
113152024-01-10T17:59:47.419Z 24 480-499 100 100 100 193 193 193 F F F
113162024-01-10T17:59:47.419Z 25 500-519 100 100 100 190 190 190 F F F
113172024-01-10T17:59:47.419Z 26 520-539 102 102 102 198 198 198 F F F
113182024-01-10T17:59:47.419Z 27 540-559 104 104 104 200 200 200 F F F
113192024-01-10T17:59:47.419Z 28 560-579 104 104 104 200 200 200 F F F
113202024-01-10T17:59:47.420Z 29 580-599 86 86 86 162 162 162 F F F
113212024-01-10T17:59:47.420ZMax gen: 104, Max flush: 200
113222024-01-10T17:59:47.423Z
113232024-01-10T17:59:47.423ZBegin loop 53
113242024-01-10T17:59:47.423ZDownstairs to restart: 1
113252024-01-10T17:59:47.450Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.453512852Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579}
113262024-01-10T17:59:47.450Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454031813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579}
113272024-01-10T17:59:47.450Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454065935Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579}
113282024-01-10T17:59:47.453Z{"msg":"Crucible 2866caa8-0458-4576-af7e-68b7e0451210 has session id: 8b9397ea-f4ef-42f6-be84-6552cea59983","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454129504Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113292024-01-10T17:59:47.453Z{"Crucible runtime is spawned
113302024-01-10T17:59:47.453Zmsg":"Upstairs opts: Upstairs UUID: 2866caa8-0458-4576-af7e-68b7e0451210, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454453718Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113312024-01-10T17:59:47.454Z{"msg":"Crucible stats registered with UUID: 2866caa8-0458-4576-af7e-68b7e0451210","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454525946Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113322024-01-10T17:59:47.454Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454552262Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579}
113332024-01-10T17:59:47.454Z{"msg":"2866caa8-0458-4576-af7e-68b7e0451210 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454728487Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113342024-01-10T17:59:47.455Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454786401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"0","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113352024-01-10T17:59:47.455Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454812865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"0","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113362024-01-10T17:59:47.455Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454841984Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"1","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113372024-01-10T17:59:47.455Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.45488371Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"1","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113382024-01-10T17:59:47.455Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.454925003Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"2","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113392024-01-10T17:59:47.455Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.45496016Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"2","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113402024-01-10T17:59:47.455Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.455026946Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113412024-01-10T17:59:47.455Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.455150535Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113422024-01-10T17:59:47.455Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.455198956Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113432024-01-10T17:59:47.455Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.455301467Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113442024-01-10T17:59:47.456Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.455683828Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113452024-01-10T17:59:47.456Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.455766878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113462024-01-10T17:59:47.456Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.456742608Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"2","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113472024-01-10T17:59:47.456Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.456805552Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"2","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113482024-01-10T17:59:47.456Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.457004116Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"0","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113492024-01-10T17:59:47.458Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.457142655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"0","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113502024-01-10T17:59:47.462Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.457644848Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"2","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113512024-01-10T17:59:47.462Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.457886224Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"1","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113522024-01-10T17:59:47.463Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.457946827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"1","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113532024-01-10T17:59:47.463Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.457984304Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"0","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113542024-01-10T17:59:47.463Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.458371188Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"2","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113552024-01-10T17:59:47.463Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.458583451Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"2","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113562024-01-10T17:59:47.463Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.458683927Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113572024-01-10T17:59:47.463Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.458864647Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"0","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113582024-01-10T17:59:47.463Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.458923457Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"0","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113592024-01-10T17:59:47.463Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.458948598Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113602024-01-10T17:59:47.463Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.458970391Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"1","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113612024-01-10T17:59:47.463Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.459646886Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"1","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113622024-01-10T17:59:47.463Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.459749423Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"1","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113632024-01-10T17:59:47.463Z{"msg":"[0]R flush_numbers[0..12]: [187, 198, 196, 198, 200, 196, 193, 200, 198, 186, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.459775863Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113642024-01-10T17:59:47.464Z{"msg":"[0]R generation[0..12]: [98, 102, 102, 102, 104, 102, 100, 104, 102, 97, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.459817451Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113652024-01-10T17:59:47.464Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.459842849Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113662024-01-10T17:59:47.464Z{"msg":"[1]R flush_numbers[0..12]: [187, 198, 196, 198, 200, 196, 193, 200, 198, 186, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.459883781Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113672024-01-10T17:59:47.464Z{"msg":"[1]R generation[0..12]: [98, 102, 102, 102, 104, 102, 100, 104, 102, 97, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.459907824Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113682024-01-10T17:59:47.464ZWait for a query_work_queue command to finish before sending IO
113692024-01-10T17:59:47.464Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.459978503Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113702024-01-10T17:59:47.464Z{"msg":"[2]R flush_numbers[0..12]: [187, 198, 196, 198, 200, 196, 193, 200, 198, 186, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460046936Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113712024-01-10T17:59:47.464Z{"msg":"[2]R generation[0..12]: [98, 102, 102, 102, 104, 102, 100, 104, 102, 97, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460074984Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113722024-01-10T17:59:47.464Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460131115Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113732024-01-10T17:59:47.464Z{"msg":"Max found gen is 105","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460158608Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113742024-01-10T17:59:47.464Z{"msg":"Generation requested: 106 >= found:105","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460199043Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113752024-01-10T17:59:47.464Z{"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460273864Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113762024-01-10T17:59:47.464Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460351385Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"":"downstairs","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113772024-01-10T17:59:47.464Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460388688Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113782024-01-10T17:59:47.464Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460455343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"0","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113792024-01-10T17:59:47.465Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460526958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"1","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113802024-01-10T17:59:47.465Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460554522Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"client":"2","":"downstairs","session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113812024-01-10T17:59:47.465Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460623371Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113822024-01-10T17:59:47.465Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460680453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113832024-01-10T17:59:47.466Z{"msg":"2866caa8-0458-4576-af7e-68b7e0451210 is now active with session: 8b9397ea-f4ef-42f6-be84-6552cea59983","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460709817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113842024-01-10T17:59:47.466Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460767396Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579,"session_id":"8b9397ea-f4ef-42f6-be84-6552cea59983"}
113852024-01-10T17:59:47.466Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:47.460791605Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1579}
113862024-01-10T17:59:47.466ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
113872024-01-10T17:59:47.466ZLoading write count information from file "/tmp/test_repair_verify.data"
113882024-01-10T17:59:47.466ZRun Repair workload
113892024-01-10T17:59:47.466Z01/30 Flush
113902024-01-10T17:59:47.466Z02/30 Write block 82 len 4096 data: 7
113912024-01-10T17:59:47.466Z03/30 Read block 423 len 24576
113922024-01-10T17:59:47.470Z04/30 Read block 36 len 20480
113932024-01-10T17:59:47.470Z05/30 Write block 453 len 20480 data: 11 10 8 7 8
113942024-01-10T17:59:47.470Z06/30 Write block 466 len 20480 data: 8 9 9 5 5
113952024-01-10T17:59:47.470Z07/30 Read block 344 len 32768
113962024-01-10T17:59:47.473Z08/30 Read block 496 len 4096
113972024-01-10T17:59:47.473Z09/30 Write block 435 len 28672 data: 2 4 4 5 7 7 9
113982024-01-10T17:59:47.473Z10/30 Read block 480 len 32768
113992024-01-10T17:59:47.476Z11/30 Read block 389 len 28672
114002024-01-10T17:59:47.479Z12/30 Flush
114012024-01-10T17:59:47.482Z13/30 Write block 280 len 4096 data: 10
114022024-01-10T17:59:47.482Z14/30 Read block 146 len 24576
114032024-01-10T17:59:47.485Z15/30 Flush
114042024-01-10T17:59:47.490Z16/30 Write block 473 len 12288 data: 4 4 3
114052024-01-10T17:59:47.490Z17/30 Read block 429 len 8192
114062024-01-10T17:59:47.490Z18/30 Read block 562 len 8192
114072024-01-10T17:59:47.495Z19/30 Write block 13 len 32768 data: 7 9 9 8 8 9 8 7
114082024-01-10T17:59:47.495Z20/30 Write block 403 len 24576 data: 10 12 14 15 14 12
114092024-01-10T17:59:47.495Z21/30 Read block 411 len 12288
114102024-01-10T17:59:47.498Z22/30 Read block 224 len 4096
114112024-01-10T17:59:47.498Z23/30 Read block 489 len 28672
114122024-01-10T17:59:47.501Z24/30 Read block 0 len 36864
114132024-01-10T17:59:47.506Z25/30 Read block 217 len 36864
114142024-01-10T17:59:47.509Z26/30 Write block 179 len 40960 data: 7 9 8 8 8 7 9 8 11 13
114152024-01-10T17:59:47.509Z27/30 Write block 244 len 12288 data: 11 10 7
114162024-01-10T17:59:47.509Z28/30 Write block 137 len 24576 data: 8 9 10 11 12 10
114172024-01-10T17:59:47.509Z29/30 Write block 358 len 32768 data: 9 8 5 5 7 10 9 8
114182024-01-10T17:59:47.514Z30/30 Write block 497 len 32768 data: 14 14 14 11 11 11 12 10
114192024-01-10T17:59:47.514Z----------------------------------------------------------------
114202024-01-10T17:59:47.514Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
114212024-01-10T17:59:47.514ZStates: Active Active Active
114222024-01-10T17:59:47.514ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
114232024-01-10T17:59:47.514Z 1 Acked 1000 Flush 0 Done Sent Done false
114242024-01-10T17:59:47.514Z 2 Acked 1001 Write 1 Done Sent Done false
114252024-01-10T17:59:47.514Z 3 Acked 1002 Read 6 Done Sent Done false
114262024-01-10T17:59:47.514Z 4 Acked 1003 Read 5 Done Sent Done false
114272024-01-10T17:59:47.514Z 5 Acked 1004 Write 5 Done Sent Done false
114282024-01-10T17:59:47.514Z 6 Acked 1005 Write 5 Done Sent Done false
114292024-01-10T17:59:47.515Z 7 Acked 1006 Read 8 Done Sent Done false
114302024-01-10T17:59:47.515Z 8 Acked 1007 Read 1 Done Sent Done false
114312024-01-10T17:59:47.515Z 9 Acked 1008 Write 7 Done Sent Done false
114322024-01-10T17:59:47.515Z 10 Acked 1009 Read 8 Done Sent Done false
114332024-01-10T17:59:47.515Z 11 Acked 1010 Read 7 Done Sent Done false
114342024-01-10T17:59:47.515Z 12 Acked 1011 Flush 0 Done Sent Done false
114352024-01-10T17:59:47.515Z 13 Acked 1012 Write 1 Done Sent Done false
114362024-01-10T17:59:47.515Z 14 Acked 1013 Read 6 Done Sent Done false
114372024-01-10T17:59:47.515Z 15 Acked 1014 Flush 0 Done Sent Done false
114382024-01-10T17:59:47.515Z 16 Acked 1015 Write 3 Done Sent Done false
114392024-01-10T17:59:47.515Z 17 Acked 1016 Read 2 Done Sent Done false
114402024-01-10T17:59:47.515Z 18 Acked 1017 Read 2 Done Sent Done false
114412024-01-10T17:59:47.515Z 19 Acked 1018 Write 8 Done Sent Done false
114422024-01-10T17:59:47.515Z 20 Acked 1019 Write 6 Done Sent Done false
114432024-01-10T17:59:47.515Z 21 Acked 1020 Read 3 Done Sent Done false
114442024-01-10T17:59:47.515Z 22 Acked 1021 Read 1 Done Sent Done false
114452024-01-10T17:59:47.515Z 23 Acked 1022 Read 7 Done Sent Done false
114462024-01-10T17:59:47.515Z 24 Acked 1023 Read 9 Done Sent Done false
114472024-01-10T17:59:47.515Z 25 Acked 1024 Read 9 Done Sent Done false
114482024-01-10T17:59:47.515Z 26 Acked 1025 Write 10 Done Sent Done false
114492024-01-10T17:59:47.516Z 27 Acked 1026 Write 3 Done Sent Done false
114502024-01-10T17:59:47.516Z 28 Acked 1027 Write 6 Done Sent Done false
114512024-01-10T17:59:47.516Z 29 Acked 1028 Write 8 Done Sent Done false
114522024-01-10T17:59:47.516Z 30 Acked 1029 Write 8 Sent Sent Sent false
114532024-01-10T17:59:47.516Z STATES DS:0 DS:1 DS:2 TOTAL
114542024-01-10T17:59:47.516Z New 0 0 0 0
114552024-01-10T17:59:47.516Z Sent 1 30 1 32
114562024-01-10T17:59:47.516Z Done 29 0 29 58
114572024-01-10T17:59:47.516Z Skipped 0 0 0 0
114582024-01-10T17:59:47.516Z Error 0 0 0 0
114592024-01-10T17:59:47.516ZLast Flush: 1014 0 1014
114602024-01-10T17:59:47.516ZDownstairs last five completed:
114612024-01-10T17:59:47.516ZUpstairs last five completed: 30 29 28 27 26
114622024-01-10T17:59:47.516ZWrote out file "/tmp/test_repair_verify.data"
114632024-01-10T17:59:47.520Z
114642024-01-10T17:59:47.526ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
114652024-01-10T17:59:47.557ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114662024-01-10T17:59:47.557Z 0 000-019 98 98 98 187 187 187 T F T <---
114672024-01-10T17:59:47.557Z 1 020-039 102 102 102 198 198 198 T F T <---
114682024-01-10T17:59:47.557Z 2 040-059 102 102 102 196 196 196 F F F
114692024-01-10T17:59:47.557Z 3 060-079 102 102 102 198 198 198 F F F
114702024-01-10T17:59:47.557Z 4 080-099 106 104 106 202 200 202 F F F <---
114712024-01-10T17:59:47.557Z 5 100-119 102 102 102 196 196 196 F F F
114722024-01-10T17:59:47.557Z 6 120-139 100 100 100 193 193 193 T F T <---
114732024-01-10T17:59:47.557Z 7 140-159 104 104 104 200 200 200 T F T <---
114742024-01-10T17:59:47.557Z 8 160-179 102 102 102 198 198 198 T F T <---
114752024-01-10T17:59:47.557Z 9 180-199 97 97 97 186 186 186 T F T <---
114762024-01-10T17:59:47.561Z 10 200-219 104 104 104 200 200 200 F F F
114772024-01-10T17:59:47.561Z 11 220-239 104 104 104 200 200 200 F F F
114782024-01-10T17:59:47.561Z 12 240-259 100 100 100 191 191 191 T F T <---
114792024-01-10T17:59:47.561Z 13 260-279 100 100 100 193 193 193 F F F
114802024-01-10T17:59:47.561Z 14 280-299 106 104 106 203 200 203 F F F <---
114812024-01-10T17:59:47.561Z 15 300-319 99 99 99 189 189 189 F F F
114822024-01-10T17:59:47.561Z 16 320-339 99 99 99 189 189 189 F F F
114832024-01-10T17:59:47.561Z 17 340-359 101 101 101 194 194 194 T F T <---
114842024-01-10T17:59:47.561Z 18 360-379 99 99 99 189 189 189 T F T <---
114852024-01-10T17:59:47.561Z 19 380-399 96 96 96 184 184 184 F F F
114862024-01-10T17:59:47.561Z 20 400-419 100 100 100 190 190 190 T F T <---
114872024-01-10T17:59:47.561Z 21 420-439 106 100 106 202 190 202 F F F <---
114882024-01-10T17:59:47.561Z 22 440-459 106 92 106 202 176 202 F F F <---
114892024-01-10T17:59:47.561Z 23 460-479 106 100 106 202 190 202 T F T <---
114902024-01-10T17:59:47.561Z 24 480-499 100 100 100 193 193 193 T F T <---
114912024-01-10T17:59:47.561Z 25 500-519 100 100 100 190 190 190 T F T <---
114922024-01-10T17:59:47.561Z 26 520-539 102 102 102 198 198 198 F F F
114932024-01-10T17:59:47.561Z 27 540-559 104 104 104 200 200 200 F F F
114942024-01-10T17:59:47.561Z 28 560-579 104 104 104 200 200 200 F F F
114952024-01-10T17:59:47.561Z 29 580-599 86 86 86 162 162 162 F F F
114962024-01-10T17:59:47.561ZMax gen: 106, Max flush: 203
114972024-01-10T17:59:47.562ZError: Difference in extent metadata found!
114982024-01-10T17:59:47.565ZA Difference in extent metadata is expected here
114992024-01-10T17:59:47.565ZOn loop 53
115002024-01-10T17:59:47.565Z
115012024-01-10T17:59:47.568ZVerifying data now
115022024-01-10T17:59:53.195ZNo change after verify
115032024-01-10T17:59:53.195ZLoop: 53 Downstairs dump after verify (and repair):
115042024-01-10T17:59:53.216ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115052024-01-10T17:59:53.216Z 0 000-019 107 107 107 204 204 204 F F F
115062024-01-10T17:59:53.216Z 1 020-039 107 107 107 204 204 204 F F F
115072024-01-10T17:59:53.216Z 2 040-059 102 102 102 196 196 196 F F F
115082024-01-10T17:59:53.216Z 3 060-079 102 102 102 198 198 198 F F F
115092024-01-10T17:59:53.217Z 4 080-099 106 106 106 202 202 202 F F F
115102024-01-10T17:59:53.217Z 5 100-119 102 102 102 196 196 196 F F F
115112024-01-10T17:59:53.217Z 6 120-139 107 107 107 204 204 204 F F F
115122024-01-10T17:59:53.217Z 7 140-159 107 107 107 204 204 204 F F F
115132024-01-10T17:59:53.217Z 8 160-179 107 107 107 204 204 204 F F F
115142024-01-10T17:59:53.217Z 9 180-199 107 107 107 204 204 204 F F F
115152024-01-10T17:59:53.217Z 10 200-219 104 104 104 200 200 200 F F F
115162024-01-10T17:59:53.217Z 11 220-239 104 104 104 200 200 200 F F F
115172024-01-10T17:59:53.217Z 12 240-259 107 107 107 204 204 204 F F F
115182024-01-10T17:59:53.217Z 13 260-279 100 100 100 193 193 193 F F F
115192024-01-10T17:59:53.217Z 14 280-299 106 106 106 203 203 203 F F F
115202024-01-10T17:59:53.217Z 15 300-319 99 99 99 189 189 189 F F F
115212024-01-10T17:59:53.223Z 16 320-339 99 99 99 189 189 189 F F F
115222024-01-10T17:59:53.223Z 17 340-359 107 107 107 204 204 204 F F F
115232024-01-10T17:59:53.223Z 18 360-379 107 107 107 204 204 204 F F F
115242024-01-10T17:59:53.223Z 19 380-399 96 96 96 184 184 184 F F F
115252024-01-10T17:59:53.223Z 20 400-419 107 107 107 204 204 204 F F F
115262024-01-10T17:59:53.223Z 21 420-439 106 106 106 202 202 202 F F F
115272024-01-10T17:59:53.223Z 22 440-459 106 106 106 202 202 202 F F F
115282024-01-10T17:59:53.223Z 23 460-479 107 107 107 204 204 204 F F F
115292024-01-10T17:59:53.223Z 24 480-499 107 107 107 204 204 204 F F F
115302024-01-10T17:59:53.223Z 25 500-519 107 107 107 204 204 204 F F F
115312024-01-10T17:59:53.223Z 26 520-539 102 102 102 198 198 198 F F F
115322024-01-10T17:59:53.223Z 27 540-559 104 104 104 200 200 200 F F F
115332024-01-10T17:59:53.223Z 28 560-579 104 104 104 200 200 200 F F F
115342024-01-10T17:59:53.223Z 29 580-599 86 86 86 162 162 162 F F F
115352024-01-10T17:59:53.224ZMax gen: 107, Max flush: 204
115362024-01-10T17:59:53.232Z
115372024-01-10T17:59:53.232ZBegin loop 54
115382024-01-10T17:59:53.232ZDownstairs to restart: 1
115392024-01-10T17:59:53.254Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.258074162Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587}
115402024-01-10T17:59:53.255Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.25862391Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587}
115412024-01-10T17:59:53.259Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.258662981Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587}
115422024-01-10T17:59:53.259Z{"msg":"Crucible 5900ce0f-f85c-40cd-b416-8ab1643ce033 has session id: 50173520-5537-467d-a8f7-b9be6d50cc93","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.258728524Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115432024-01-10T17:59:53.259Z{"msg":"Upstairs opts: Upstairs UUID: 5900ce0f-f85c-40cd-b416-8ab1643ce033, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.25878372Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115442024-01-10T17:59:53.259Z{"msg":"Crucible stats registered with UUID: 5900ce0f-f85c-40cd-b416-8ab1643ce033","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.258862096Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115452024-01-10T17:59:53.259ZCrucible runtime is spawned
115462024-01-10T17:59:53.259Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259168438Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587}
115472024-01-10T17:59:53.259Z{"msg":"5900ce0f-f85c-40cd-b416-8ab1643ce033 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259267606Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115482024-01-10T17:59:53.259Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259380683Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115492024-01-10T17:59:53.259Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259471412Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115502024-01-10T17:59:53.260Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259553855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115512024-01-10T17:59:53.260Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259599607Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115522024-01-10T17:59:53.261Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259693975Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115532024-01-10T17:59:53.261Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259723017Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115542024-01-10T17:59:53.261Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259761448Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115552024-01-10T17:59:53.261Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259910441Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115562024-01-10T17:59:53.261Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.259955542Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115572024-01-10T17:59:53.261Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.260011491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115582024-01-10T17:59:53.261Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.260053939Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115592024-01-10T17:59:53.262Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.260111634Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115602024-01-10T17:59:53.262Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.261439452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115612024-01-10T17:59:53.262Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.261526323Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115622024-01-10T17:59:53.263Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.26167257Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115632024-01-10T17:59:53.263Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.261743488Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115642024-01-10T17:59:53.263Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.262543422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115652024-01-10T17:59:53.263Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.262833271Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115662024-01-10T17:59:53.268Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.263321366Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115672024-01-10T17:59:53.268Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.263389211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115682024-01-10T17:59:53.268Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.263684862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115692024-01-10T17:59:53.268Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.263768266Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115702024-01-10T17:59:53.268Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.26394021Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115712024-01-10T17:59:53.268Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.263980938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115722024-01-10T17:59:53.268Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.264020899Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115732024-01-10T17:59:53.268Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.264081366Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115742024-01-10T17:59:53.268Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.264338924Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115752024-01-10T17:59:53.268Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.26504973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115762024-01-10T17:59:53.268Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265149787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115772024-01-10T17:59:53.268Z{"msg":"[0]R flush_numbers[0..12]: [204, 204, 196, 198, 202, 196, 204, 204, 204, 204, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265220648Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115782024-01-10T17:59:53.268Z{"msg":"[0]R generation[0..12]: [107, 107, 102, 102, 106, 102, 107, 107, 107, 107, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265249343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115792024-01-10T17:59:53.268Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265314484Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115802024-01-10T17:59:53.268ZWait for a query_work_queue command to finish before sending IO
115812024-01-10T17:59:53.268Z{"msg":"[1]R flush_numbers[0..12]: [204, 204, 196, 198, 202, 196, 204, 204, 204, 204, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265349301Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115822024-01-10T17:59:53.268Z{"msg":"[1]R generation[0..12]: [107, 107, 102, 102, 106, 102, 107, 107, 107, 107, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265404507Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115832024-01-10T17:59:53.268Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265431143Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115842024-01-10T17:59:53.269Z{"msg":"[2]R flush_numbers[0..12]: [204, 204, 196, 198, 202, 196, 204, 204, 204, 204, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.26546019Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115852024-01-10T17:59:53.269Z{"msg":"[2]R generation[0..12]: [107, 107, 102, 102, 106, 102, 107, 107, 107, 107, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265515586Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115862024-01-10T17:59:53.269Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265542347Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115872024-01-10T17:59:53.269Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265598108Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115882024-01-10T17:59:53.269Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265624195Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115892024-01-10T17:59:53.269Z{"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265679625Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115902024-01-10T17:59:53.279Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265705356Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115912024-01-10T17:59:53.279Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265761256Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115922024-01-10T17:59:53.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265785503Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115932024-01-10T17:59:53.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265840802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115942024-01-10T17:59:53.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265866316Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115952024-01-10T17:59:53.279Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265921326Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115962024-01-10T17:59:53.279Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.265945509Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115972024-01-10T17:59:53.279ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
115982024-01-10T17:59:53.279Z{"msg":"5900ce0f-f85c-40cd-b416-8ab1643ce033 is now active with session: 50173520-5537-467d-a8f7-b9be6d50cc93","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.266006127Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
115992024-01-10T17:59:53.279Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.266029439Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587,"session_id":"50173520-5537-467d-a8f7-b9be6d50cc93"}
116002024-01-10T17:59:53.279Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:53.266055508Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1587}
116012024-01-10T17:59:53.279ZLoading write count information from file "/tmp/test_repair_verify.data"
116022024-01-10T17:59:53.279ZRun Repair workload
116032024-01-10T17:59:53.279Z01/30 Write block 206 len 32768 data: 4 5 5 7 7 8 8 8
116042024-01-10T17:59:53.279Z02/30 Read block 156 len 12288
116052024-01-10T17:59:53.313Z03/30 Write block 266 len 28672 data: 7 9 7 8 9 9 7
116062024-01-10T17:59:53.313Z04/30 Read block 294 len 32768
116072024-01-10T17:59:53.313Z05/30 Write block 516 len 16384 data: 12 11 10 9
116082024-01-10T17:59:53.313Z06/30 Read block 572 len 40960
116092024-01-10T17:59:53.313Z07/30 Read block 499 len 12288
116102024-01-10T17:59:53.313Z08/30 Write block 196 len 24576 data: 13 13 14 14 13 12
116112024-01-10T17:59:53.313Z09/30 Read block 180 len 32768
116122024-01-10T17:59:53.317Z10/30 Write block 541 len 40960 data: 10 7 8 9 10 10 14 14 15 16
116132024-01-10T17:59:53.317Z11/30 Read block 290 len 20480
116142024-01-10T17:59:53.318Z12/30 Read block 433 len 12288
116152024-01-10T17:59:53.318Z13/30 Read block 52 len 40960
116162024-01-10T17:59:53.318Z14/30 Read block 292 len 12288
116172024-01-10T17:59:53.318Z15/30 Write block 219 len 36864 data: 7 7 9 8 6 5 7 6 7
116182024-01-10T17:59:53.318Z16/30 Read block 487 len 12288
116192024-01-10T17:59:53.318Z17/30 Flush
116202024-01-10T17:59:53.318Z18/30 Write block 77 len 4096 data: 9
116212024-01-10T17:59:53.318Z19/30 Read block 212 len 20480
116222024-01-10T17:59:53.318Z20/30 Flush
116232024-01-10T17:59:53.318Z21/30 Flush
116242024-01-10T17:59:53.318Z22/30 Write block 447 len 28672 data: 11 9 9 9 7 9 12
116252024-01-10T17:59:53.318Z23/30 Read block 294 len 24576
116262024-01-10T17:59:53.318Z24/30 Write block 208 len 28672 data: 6 8 8 9 9 9 8
116272024-01-10T17:59:53.318Z25/30 Read block 8 len 20480
116282024-01-10T17:59:53.318Z26/30 Write block 204 len 20480 data: 7 6 5 6 7
116292024-01-10T17:59:53.318Z27/30 Write block 72 len 24576 data: 3 4 6 9 10 10
116302024-01-10T17:59:53.318Z28/30 Write block 195 len 36864 data: 15 14 14 15 15 14 13 11 9
116312024-01-10T17:59:53.318Z29/30 Write block 161 len 4096 data: 8
116322024-01-10T17:59:53.323Z30/30 Read block 121 len 36864
116332024-01-10T17:59:53.323Z----------------------------------------------------------------
116342024-01-10T17:59:53.323Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
116352024-01-10T17:59:53.323ZStates: Active Active Active
116362024-01-10T17:59:53.323ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
116372024-01-10T17:59:53.323Z 1 Acked 1000 Write 8 Done Done Done false
116382024-01-10T17:59:53.323Z 2 Acked 1001 Read 3 Done Done Done false
116392024-01-10T17:59:53.323Z 3 Acked 1002 Write 7 Done Sent Done false
116402024-01-10T17:59:53.323Z 4 Acked 1003 Read 8 Done Sent Done false
116412024-01-10T17:59:53.323Z 5 Acked 1004 Write 4 Done Sent Done false
116422024-01-10T17:59:53.323Z 6 Acked 1005 Read 10 Done Sent Done false
116432024-01-10T17:59:53.323Z 7 Acked 1006 Read 3 Done Sent Done false
116442024-01-10T17:59:53.323Z 8 Acked 1007 Write 6 Done Sent Done false
116452024-01-10T17:59:53.323Z 9 Acked 1008 Read 8 Done Sent Done false
116462024-01-10T17:59:53.323Z 10 Acked 1009 Write 10 Done Sent Done false
116472024-01-10T17:59:53.323Z 11 Acked 1010 Read 5 Done Sent Done false
116482024-01-10T17:59:53.323Z 12 Acked 1011 Read 3 Done Sent Done false
116492024-01-10T17:59:53.324Z 13 Acked 1012 Read 10 Done Sent Done false
116502024-01-10T17:59:53.324Z 14 Acked 1013 Read 3 Done Sent Done false
116512024-01-10T17:59:53.324Z 15 Acked 1014 Write 9 Done Sent Done false
116522024-01-10T17:59:53.324Z 16 Acked 1015 Read 3 Done Sent Done false
116532024-01-10T17:59:53.324Z 17 Acked 1016 Flush 0 Done Sent Done false
116542024-01-10T17:59:53.325Z 18 Acked 1017 Write 1 Done Sent Done false
116552024-01-10T17:59:53.325Z 19 Acked 1018 Read 5 Done Sent Done false
116562024-01-10T17:59:53.325Z 20 Acked 1019 Flush 0 Done Sent Done false
116572024-01-10T17:59:53.325Z 21 Acked 1020 Flush 0 Done Sent Done false
116582024-01-10T17:59:53.325Z 22 Acked 1021 Write 7 Done Sent Done false
116592024-01-10T17:59:53.325Z 23 Acked 1022 Read 6 Done Sent Done false
116602024-01-10T17:59:53.325Z 24 Acked 1023 Write 7 Done Sent Done false
116612024-01-10T17:59:53.325Z 25 Acked 1024 Read 5 Done Sent Done false
116622024-01-10T17:59:53.325Z 26 Acked 1025 Write 5 Done Sent Done false
116632024-01-10T17:59:53.325Z 27 Acked 1026 Write 6 Done Sent Done false
116642024-01-10T17:59:53.326Z 28 Acked 1027 Write 9 Done Sent Done false
116652024-01-10T17:59:53.326Z 29 Acked 1028 Write 1 Done Sent Sent false
116662024-01-10T17:59:53.326Z 30 Acked 1029 Read 9 Done Sent Sent false
116672024-01-10T17:59:53.326Z STATES DS:0 DS:1 DS:2 TOTAL
116682024-01-10T17:59:53.326Z New 0 0 0 0
116692024-01-10T17:59:53.326Z Sent 0 28 2 30
116702024-01-10T17:59:53.326Z Done 30 2 28 60
116712024-01-10T17:59:53.326Z Skipped 0 0 0 0
116722024-01-10T17:59:53.326Z Error 0 0 0 0
116732024-01-10T17:59:53.327ZLast Flush: 1020 0 1020
116742024-01-10T17:59:53.327ZDownstairs last five completed:
116752024-01-10T17:59:53.327ZUpstairs last five completed: 30 29 28 27 26
116762024-01-10T17:59:53.327ZWrote out file "/tmp/test_repair_verify.data"
116772024-01-10T17:59:53.331Z
116782024-01-10T17:59:53.335ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
116792024-01-10T17:59:53.361ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116802024-01-10T17:59:53.361Z 0 000-019 107 107 107 204 204 204 F F F
116812024-01-10T17:59:53.361Z 1 020-039 107 107 107 204 204 204 F F F
116822024-01-10T17:59:53.361Z 2 040-059 102 102 102 196 196 196 F F F
116832024-01-10T17:59:53.361Z 3 060-079 108 102 108 206 198 206 T F T <---
116842024-01-10T17:59:53.361Z 4 080-099 106 106 106 202 202 202 F F F
116852024-01-10T17:59:53.361Z 5 100-119 102 102 102 196 196 196 F F F
116862024-01-10T17:59:53.362Z 6 120-139 107 107 107 204 204 204 F F F
116872024-01-10T17:59:53.362Z 7 140-159 107 107 107 204 204 204 F F F
116882024-01-10T17:59:53.362Z 8 160-179 107 107 107 204 204 204 T F T <---
116892024-01-10T17:59:53.365Z 9 180-199 108 107 108 205 204 205 T F T <---
116902024-01-10T17:59:53.365Z 10 200-219 108 104 108 205 200 205 T T T <---
116912024-01-10T17:59:53.365Z 11 220-239 108 104 108 205 200 205 F F F <---
116922024-01-10T17:59:53.365Z 12 240-259 107 107 107 204 204 204 F F F
116932024-01-10T17:59:53.365Z 13 260-279 108 100 108 205 193 205 F F F <---
116942024-01-10T17:59:53.365Z 14 280-299 106 106 106 203 203 203 F F F
116952024-01-10T17:59:53.365Z 15 300-319 99 99 99 189 189 189 F F F
116962024-01-10T17:59:53.365Z 16 320-339 99 99 99 189 189 189 F F F
116972024-01-10T17:59:53.366Z 17 340-359 107 107 107 204 204 204 F F F
116982024-01-10T17:59:53.366Z 18 360-379 107 107 107 204 204 204 F F F
116992024-01-10T17:59:53.366Z 19 380-399 96 96 96 184 184 184 F F F
117002024-01-10T17:59:53.366Z 20 400-419 107 107 107 204 204 204 F F F
117012024-01-10T17:59:53.366Z 21 420-439 106 106 106 202 202 202 F F F
117022024-01-10T17:59:53.366Z 22 440-459 106 106 106 202 202 202 T F T <---
117032024-01-10T17:59:53.366Z 23 460-479 107 107 107 204 204 204 F F F
117042024-01-10T17:59:53.366Z 24 480-499 107 107 107 204 204 204 F F F
117052024-01-10T17:59:53.366Z 25 500-519 108 107 108 205 204 205 F F F <---
117062024-01-10T17:59:53.366Z 26 520-539 102 102 102 198 198 198 F F F
117072024-01-10T17:59:53.366Z 27 540-559 108 104 108 205 200 205 F F F <---
117082024-01-10T17:59:53.367Z 28 560-579 104 104 104 200 200 200 F F F
117092024-01-10T17:59:53.367Z 29 580-599 86 86 86 162 162 162 F F F
117102024-01-10T17:59:53.367ZMax gen: 108, Max flush: 206
117112024-01-10T17:59:53.367ZError: Difference in extent metadata found!
117122024-01-10T17:59:53.370ZA Difference in extent metadata is expected here
117132024-01-10T17:59:53.370ZOn loop 54
117142024-01-10T17:59:53.370Z
117152024-01-10T17:59:53.373ZVerifying data now
117162024-01-10T17:59:58.367ZNo change after verify
117172024-01-10T17:59:58.367ZLoop: 54 Downstairs dump after verify (and repair):
117182024-01-10T17:59:58.390ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117192024-01-10T17:59:58.390Z 0 000-019 107 107 107 204 204 204 F F F
117202024-01-10T17:59:58.390Z 1 020-039 107 107 107 204 204 204 F F F
117212024-01-10T17:59:58.390Z 2 040-059 102 102 102 196 196 196 F F F
117222024-01-10T17:59:58.390Z 3 060-079 109 109 109 207 207 207 F F F
117232024-01-10T17:59:58.390Z 4 080-099 106 106 106 202 202 202 F F F
117242024-01-10T17:59:58.390Z 5 100-119 102 102 102 196 196 196 F F F
117252024-01-10T17:59:58.390Z 6 120-139 107 107 107 204 204 204 F F F
117262024-01-10T17:59:58.390Z 7 140-159 107 107 107 204 204 204 F F F
117272024-01-10T17:59:58.390Z 8 160-179 109 109 109 207 207 207 F F F
117282024-01-10T17:59:58.390Z 9 180-199 109 109 109 207 207 207 F F F
117292024-01-10T17:59:58.390Z 10 200-219 109 109 109 207 207 207 F F F
117302024-01-10T17:59:58.397Z 11 220-239 108 108 108 205 205 205 F F F
117312024-01-10T17:59:58.397Z 12 240-259 107 107 107 204 204 204 F F F
117322024-01-10T17:59:58.397Z 13 260-279 108 108 108 205 205 205 F F F
117332024-01-10T17:59:58.398Z 14 280-299 106 106 106 203 203 203 F F F
117342024-01-10T17:59:58.398Z 15 300-319 99 99 99 189 189 189 F F F
117352024-01-10T17:59:58.398Z 16 320-339 99 99 99 189 189 189 F F F
117362024-01-10T17:59:58.398Z 17 340-359 107 107 107 204 204 204 F F F
117372024-01-10T17:59:58.398Z 18 360-379 107 107 107 204 204 204 F F F
117382024-01-10T17:59:58.398Z 19 380-399 96 96 96 184 184 184 F F F
117392024-01-10T17:59:58.398Z 20 400-419 107 107 107 204 204 204 F F F
117402024-01-10T17:59:58.398Z 21 420-439 106 106 106 202 202 202 F F F
117412024-01-10T17:59:58.398Z 22 440-459 109 109 109 207 207 207 F F F
117422024-01-10T17:59:58.399Z 23 460-479 107 107 107 204 204 204 F F F
117432024-01-10T17:59:58.399Z 24 480-499 107 107 107 204 204 204 F F F
117442024-01-10T17:59:58.399Z 25 500-519 108 108 108 205 205 205 F F F
117452024-01-10T17:59:58.399Z 26 520-539 102 102 102 198 198 198 F F F
117462024-01-10T17:59:58.399Z 27 540-559 108 108 108 205 205 205 F F F
117472024-01-10T17:59:58.399Z 28 560-579 104 104 104 200 200 200 F F F
117482024-01-10T17:59:58.399Z 29 580-599 86 86 86 162 162 162 F F F
117492024-01-10T17:59:58.399ZMax gen: 109, Max flush: 207
117502024-01-10T17:59:58.399Z
117512024-01-10T17:59:58.399ZBegin loop 55
117522024-01-10T17:59:58.399ZDownstairs to restart: 2
117532024-01-10T17:59:58.426Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.430074599Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595}
117542024-01-10T17:59:58.426Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.430613291Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595}
117552024-01-10T17:59:58.432Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.430687233Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595}
117562024-01-10T17:59:58.432Z{"msg":"Crucible 13103812-b044-418e-b287-3d250c91e3c6 has session id: 393005b8-8f9c-4976-b098-e09a6bc94324","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.430765175Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117572024-01-10T17:59:58.432Z{"msg":"Upstairs opts: Upstairs UUID: 13103812-b044-418e-b287-3d250c91e3c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
117582024-01-10T17:59:58.432Z,"time":"2024-01-10T17:59:58.430824028Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117592024-01-10T17:59:58.432Z{"msg":"Crucible stats registered with UUID: 13103812-b044-418e-b287-3d250c91e3c6","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.430964159Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117602024-01-10T17:59:58.432Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.431123132Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595}
117612024-01-10T17:59:58.432Z{"msg":"13103812-b044-418e-b287-3d250c91e3c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.431288801Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117622024-01-10T17:59:58.432Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.431382448Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117632024-01-10T17:59:58.433Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.431430289Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117642024-01-10T17:59:58.433Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.431531663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117652024-01-10T17:59:58.433Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.431582613Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117662024-01-10T17:59:58.433Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.431624696Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117672024-01-10T17:59:58.433Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.431696002Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117682024-01-10T17:59:58.433Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.431983606Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117692024-01-10T17:59:58.434Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.432013736Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117702024-01-10T17:59:58.434Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.432037203Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117712024-01-10T17:59:58.434Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.432073693Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117722024-01-10T17:59:58.434Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.432131613Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117732024-01-10T17:59:58.434Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.432219332Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117742024-01-10T17:59:58.434Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.433554215Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117752024-01-10T17:59:58.434Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.433654031Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117762024-01-10T17:59:58.434Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.433879649Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117772024-01-10T17:59:58.434Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.433939265Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117782024-01-10T17:59:58.434Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.434429605Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117792024-01-10T17:59:58.434Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.434631692Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117802024-01-10T17:59:58.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.434732601Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117812024-01-10T17:59:58.435Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.434820683Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117822024-01-10T17:59:58.435Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.435585042Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117832024-01-10T17:59:58.435Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.435760006Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117842024-01-10T17:59:58.441Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.435798406Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117852024-01-10T17:59:58.441Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.435857837Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117862024-01-10T17:59:58.441Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.43588453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117872024-01-10T17:59:58.441Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.435907259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117882024-01-10T17:59:58.441Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.435929003Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117892024-01-10T17:59:58.442Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.436753504Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117902024-01-10T17:59:58.442Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.436815508Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117912024-01-10T17:59:58.442Z{"msg":"[0]R flush_numbers[0..12]: [204, 204, 196, 207, 202, 196, 204, 204, 207, 207, 207, 205]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.436842999Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117922024-01-10T17:59:58.442Z{"msg":"[0]R generation[0..12]: [107, 107, 102, 109, 106, 102, 107, 107, 109, 109, 109, 108]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.436885238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117932024-01-10T17:59:58.442Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.436910343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117942024-01-10T17:59:58.442Z{"msg":"[1]R flush_numbers[0..12]: [204, 204, 196, 207, 202, 196, 204, 204, 207, 207, 207, 205]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.436933569Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117952024-01-10T17:59:58.452Z{"msg":"[1]R generation[0..12]: [107, 107, 102, 109, 106, 102, 107, 107, 109, 109, 109, 108]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
117962024-01-10T17:59:58.452Z,"time":"2024-01-10T17:59:58.436962095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117972024-01-10T17:59:58.452Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437077317Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117982024-01-10T17:59:58.452Z{"msg":"[2]R flush_numbers[0..12]: [204, 204, 196, 207, 202, 196, 204, 204, 207, 207, 207, 205]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437108066Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
117992024-01-10T17:59:58.452Z{"msg":"[2]R generation[0..12]: [107, 107, 102, 109, 106, 102, 107, 107, 109, 109, 109, 108]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437163527Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118002024-01-10T17:59:58.452Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437191607Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118012024-01-10T17:59:58.452Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437248117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118022024-01-10T17:59:58.452Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437290472Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118032024-01-10T17:59:58.452Z{"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437347127Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118042024-01-10T17:59:58.453Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437389049Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118052024-01-10T17:59:58.453Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437443935Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118062024-01-10T17:59:58.453Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437482141Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118072024-01-10T17:59:58.453Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437537762Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118082024-01-10T17:59:58.453Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437579536Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118092024-01-10T17:59:58.453Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437642979Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118102024-01-10T17:59:58.453Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.43768143Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118112024-01-10T17:59:58.453Z{"msg":"13103812-b044-418e-b287-3d250c91e3c6 is now active with session: 393005b8-8f9c-4976-b098-e09a6bc94324","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118122024-01-10T17:59:58.453Zcrucible","level":30,"time":"2024-01-10T17:59:58.437773818Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118132024-01-10T17:59:58.453Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.43782765Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595,"session_id":"393005b8-8f9c-4976-b098-e09a6bc94324"}
118142024-01-10T17:59:58.453Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-01-10T17:59:58.437866604Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1595}
118152024-01-10T17:59:58.453ZLoading write count information from file "/tmp/test_repair_verify.data"
118162024-01-10T17:59:58.453ZRun Repair workload
118172024-01-10T17:59:58.454Z01/30 Write block 91 len 12288 data: 9 8 8
118182024-01-10T17:59:58.454Z02/30 Read block 355 len 24576
118192024-01-10T17:59:58.454Z03/30 Read block 591 len 8192
118202024-01-10T17:59:58.487Z04/30 Write block 90 len 36864 data: 7 10 9 9 9 12 11 11 8
118212024-01-10T17:59:58.487Z05/30 Flush
118222024-01-10T17:59:58.487Z06/30 Write block 278 len 4096 data: 9
118232024-01-10T17:59:58.487Z07/30 Write block 557 len 28672 data: 10 8 8 9 10 10 10
118242024-01-10T17:59:58.487Z08/30 Read block 430 len 40960
118252024-01-10T17:59:58.487Z09/30 Write block 318 len 16384 data: 6 7 7 8
118262024-01-10T17:59:58.487Z10/30 Write block 202 len 28672 data: 12 10 8 7 6 7 8
118272024-01-10T17:59:58.487Z11/30 Write block 579 len 36864 data: 7 7 8 8 9 7 6 8 7
118282024-01-10T17:59:58.487Z12/30 Read block 162 len 16384
118292024-01-10T17:59:58.487Z13/30 Read block 528 len 32768
118302024-01-10T17:59:58.490Z14/30 Write block 512 len 4096 data: 12
118312024-01-10T17:59:58.491Z15/30 Write block 354 len 24576 data: 7 5 6 7 10 9
118322024-01-10T17:59:58.491Z16/30 Write block 589 len 16384 data: 7 7 7 6
118332024-01-10T17:59:58.491Z17/30 Write block 419 len 12288 data: 13 16 13
118342024-01-10T17:59:58.491Z18/30 Read block 122 len 16384
118352024-01-10T17:59:58.491Z19/30 Read block 529 len 32768
118362024-01-10T17:59:58.491Z20/30 Write block 584 len 24576 data: 8 7 9 8 7 8
118372024-01-10T17:59:58.491Z21/30 Read block 110 len 36864
118382024-01-10T17:59:58.491Z22/30 Read block 69 len 36864
118392024-01-10T17:59:58.491Z23/30 Read block 262 len 20480
118402024-01-10T17:59:58.491Z24/30 Read block 189 len 20480
118412024-01-10T17:59:58.491Z25/30 Flush
118422024-01-10T17:59:58.491Z26/30 Write block 237 len 36864 data: 7 7 8 8 7 7 8 12 11
118432024-01-10T17:59:58.492Z27/30 Write block 533 len 40960 data: 12 12 11 11 9 9 8 10 11 8
118442024-01-10T17:59:58.492Z28/30 Write block 453 len 32768 data: 13 11 9 8 9 6 6 7
118452024-01-10T17:59:58.492Z29/30 Write block 474 len 8192 data: 5 4
118462024-01-10T17:59:58.492Z30/30 Read block 553 len 16384
118472024-01-10T17:59:58.496Z----------------------------------------------------------------
118482024-01-10T17:59:58.496Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
118492024-01-10T17:59:58.496ZStates: Active Active Active
118502024-01-10T17:59:58.496ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
118512024-01-10T17:59:58.496Z 1 Acked 1000 Write 3 Done Done Done false
118522024-01-10T17:59:58.496Z 2 Acked 1001 Read 6 Done Done Sent false
118532024-01-10T17:59:58.496Z 3 Acked 1002 Read 2 Done Done Sent false
118542024-01-10T17:59:58.496Z 4 Acked 1003 Write 9 Done Done Sent false
118552024-01-10T17:59:58.496Z 5 Acked 1004 Flush 0 Done Done Sent false
118562024-01-10T17:59:58.496Z 6 Acked 1005 Write 1 Done Done Sent false
118572024-01-10T17:59:58.496Z 7 Acked 1006 Write 7 Done Done Sent false
118582024-01-10T17:59:58.496Z 8 Acked 1007 Read 10 Done Done Sent false
118592024-01-10T17:59:58.496Z 9 Acked 1008 Write 4 Done Done Sent false
118602024-01-10T17:59:58.497Z 10 Acked 1009 Write 7 Done Done Sent false
118612024-01-10T17:59:58.497Z 11 Acked 1010 Write 9 Done Done Sent false
118622024-01-10T17:59:58.497Z 12 Acked 1011 Read 4 Done Done Sent false
118632024-01-10T17:59:58.497Z 13 Acked 1012 Read 8 Done Done Sent false
118642024-01-10T17:59:58.497Z 14 Acked 1013 Write 1 Done Done Sent false
118652024-01-10T17:59:58.497Z 15 Acked 1014 Write 6 Done Done Sent false
118662024-01-10T17:59:58.497Z 16 Acked 1015 Write 4 Done Done Sent false
118672024-01-10T17:59:58.497Z 17 Acked 1016 Write 3 Done Done Sent false
118682024-01-10T17:59:58.497Z 18 Acked 1017 Read 4 Done Done Sent false
118692024-01-10T17:59:58.498Z 19 Acked 1018 Read 8 Done Done Sent false
118702024-01-10T17:59:58.498Z 20 Acked 1019 Write 6 Done Done Sent false
118712024-01-10T17:59:58.498Z 21 Acked 1020 Read 9 Done Done Sent false
118722024-01-10T17:59:58.498Z 22 Acked 1021 Read 9 Done Done Sent false
118732024-01-10T17:59:58.498Z 23 Acked 1022 Read 5 Done Done Sent false
118742024-01-10T17:59:58.498Z 24 Acked 1023 Read 5 Done Done Sent false
118752024-01-10T17:59:58.498Z 25 Acked 1024 Flush 0 Done Done Sent false
118762024-01-10T17:59:58.498Z 26 Acked 1025 Write 9 Done Done Sent false
118772024-01-10T17:59:58.498Z 27 Acked 1026 Write 10 Done Done Sent false
118782024-01-10T17:59:58.499Z 28 Acked 1027 Write 8 Done Done Sent false
118792024-01-10T17:59:58.499Z 29 Acked 1028 Write 2 Done Done Sent false
118802024-01-10T17:59:58.499Z 30 Acked 1029 Read 4 Done Done Sent false
118812024-01-10T17:59:58.499Z STATES DS:0 DS:1 DS:2 TOTAL
118822024-01-10T17:59:58.499Z New 0 0 0 0
118832024-01-10T17:59:58.499Z Sent 0 0 29 29
118842024-01-10T17:59:58.499Z Done 30 30 1 61
118852024-01-10T17:59:58.499Z Skipped 0 0 0 0
118862024-01-10T17:59:58.499Z Error 0 0 0 0
118872024-01-10T17:59:58.499ZLast Flush: 1024 1024 0
118882024-01-10T17:59:58.499ZDownstairs last five completed:
118892024-01-10T17:59:58.499ZUpstairs last five completed: 30 29 28 27 26
118902024-01-10T17:59:58.500ZWrote out file "/tmp/test_repair_verify.data"
118912024-01-10T17:59:58.503Z
118922024-01-10T17:59:58.507ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
118932024-01-10T17:59:58.533ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118942024-01-10T17:59:58.533Z 0 000-019 107 107 107 204 204 204 F F F
118952024-01-10T17:59:58.533Z 1 020-039 107 107 107 204 204 204 F F F
118962024-01-10T17:59:58.533Z 2 040-059 102 102 102 196 196 196 F F F
118972024-01-10T17:59:58.533Z 3 060-079 109 109 109 207 207 207 F F F
118982024-01-10T17:59:58.533Z 4 080-099 110 110 106 208 208 202 F F T <---
118992024-01-10T17:59:58.533Z 5 100-119 102 102 102 196 196 196 F F F
119002024-01-10T17:59:58.533Z 6 120-139 107 107 107 204 204 204 F F F
119012024-01-10T17:59:58.537Z 7 140-159 107 107 107 204 204 204 F F F
119022024-01-10T17:59:58.537Z 8 160-179 109 109 109 207 207 207 F F F
119032024-01-10T17:59:58.537Z 9 180-199 109 109 109 207 207 207 F F F
119042024-01-10T17:59:58.537Z 10 200-219 110 110 109 209 209 207 F F F <---
119052024-01-10T17:59:58.537Z 11 220-239 108 108 108 205 205 205 T T F <---
119062024-01-10T17:59:58.537Z 12 240-259 107 107 107 204 204 204 T T F <---
119072024-01-10T17:59:58.537Z 13 260-279 110 110 108 209 209 205 F F F <---
119082024-01-10T17:59:58.537Z 14 280-299 106 106 106 203 203 203 F F F
119092024-01-10T17:59:58.537Z 15 300-319 110 110 99 209 209 189 F F F <---
119102024-01-10T17:59:58.537Z 16 320-339 110 110 99 209 209 189 F F F <---
119112024-01-10T17:59:58.537Z 17 340-359 110 110 107 209 209 204 F F F <---
119122024-01-10T17:59:58.537Z 18 360-379 107 107 107 204 204 204 F F F
119132024-01-10T17:59:58.537Z 19 380-399 96 96 96 184 184 184 F F F
119142024-01-10T17:59:58.540Z 20 400-419 110 110 107 209 209 204 F F F <---
119152024-01-10T17:59:58.540Z 21 420-439 110 110 106 209 209 202 F F F <---
119162024-01-10T17:59:58.540Z 22 440-459 109 109 109 207 207 207 T T F <---
119172024-01-10T17:59:58.540Z 23 460-479 107 107 107 204 204 204 T T F <---
119182024-01-10T17:59:58.540Z 24 480-499 107 107 107 204 204 204 F F F
119192024-01-10T17:59:58.540Z 25 500-519 110 110 108 209 209 205 F F F <---
119202024-01-10T17:59:58.540Z 26 520-539 102 102 102 198 198 198 T T F <---
119212024-01-10T17:59:58.540Z 27 540-559 110 110 108 209 209 205 T T F <---
119222024-01-10T17:59:58.540Z 28 560-579 110 110 104 209 209 200 F F F <---
119232024-01-10T17:59:58.540Z 29 580-599 110 110 86 209 209 162 F F F <---
119242024-01-10T17:59:58.540ZMax gen: 110, Max flush: 209
119252024-01-10T17:59:58.540ZError: Difference in extent metadata found!
119262024-01-10T17:59:58.544ZA Difference in extent metadata is expected here
119272024-01-10T17:59:58.544ZOn loop 55
119282024-01-10T17:59:58.544Z
119292024-01-10T17:59:58.544ZVerifying data now
119302024-01-10T18:00:04.132ZNo change after verify
119312024-01-10T18:00:04.133ZLoop: 55 Downstairs dump after verify (and repair):
119322024-01-10T18:00:04.154ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119332024-01-10T18:00:04.155Z 0 000-019 107 107 107 204 204 204 F F F
119342024-01-10T18:00:04.155Z 1 020-039 107 107 107 204 204 204 F F F
119352024-01-10T18:00:04.155Z 2 040-059 102 102 102 196 196 196 F F F
119362024-01-10T18:00:04.155Z 3 060-079 109 109 109 207 207 207 F F F
119372024-01-10T18:00:04.155Z 4 080-099 110 110 110 208 208 208 F F F
119382024-01-10T18:00:04.155Z 5 100-119 102 102 102 196 196 196 F F F
119392024-01-10T18:00:04.159Z 6 120-139 107 107 107 204 204 204 F F F
119402024-01-10T18:00:04.159Z 7 140-159 107 107 107 204 204 204 F F F
119412024-01-10T18:00:04.159Z 8 160-179 109 109 109 207 207 207 F F F
119422024-01-10T18:00:04.160Z 9 180-199 109 109 109 207 207 207 F F F
119432024-01-10T18:00:04.160Z 10 200-219 110 110 110 209 209 209 F F F
119442024-01-10T18:00:04.160Z 11 220-239 111 111 111 210 210 210 F F F
119452024-01-10T18:00:04.160Z 12 240-259 111 111 111 210 210 210 F F F
119462024-01-10T18:00:04.160Z 13 260-279 110 110 110 209 209 209 F F F
119472024-01-10T18:00:04.160Z 14 280-299 106 106 106 203 203 203 F F F
119482024-01-10T18:00:04.160Z 15 300-319 110 110 110 209 209 209 F F F
119492024-01-10T18:00:04.160Z 16 320-339 110 110 110 209 209 209 F F F
119502024-01-10T18:00:04.160Z 17 340-359 110 110 110 209 209 209 F F F
119512024-01-10T18:00:04.160Z 18 360-379 107 107 107 204 204 204 F F F
119522024-01-10T18:00:04.160Z 19 380-399 96 96 96 184 184 184 F F F
119532024-01-10T18:00:04.160Z 20 400-419 110 110 110 209 209 209 F F F
119542024-01-10T18:00:04.161Z 21 420-439 110 110 110 209 209 209 F F F
119552024-01-10T18:00:04.161Z 22 440-459 111 111 111 210 210 210 F F F
119562024-01-10T18:00:04.161Z 23 460-479 111 111 111 210 210 210 F F F
119572024-01-10T18:00:04.161Z 24 480-499 107 107 107 204 204 204 F F F
119582024-01-10T18:00:04.161Z 25 500-519 110 110 110 209 209 209 F F F
119592024-01-10T18:00:04.161Z 26 520-539 111 111 111 210 210 210 F F F
119602024-01-10T18:00:04.161Z 27 540-559 111 111 111 210 210 210 F F F
119612024-01-10T18:00:04.161Z 28 560-579 110 110 110 209 209 209 F F F
119622024-01-10T18:00:04.161Z 29 580-599 110 110 110 209 209 209 F F F
119632024-01-10T18:00:04.161ZMax gen: 111, Max flush: 210
119642024-01-10T18:00:04.161Z
119652024-01-10T18:00:04.161ZBegin loop 56
119662024-01-10T18:00:04.161ZDownstairs to restart: 1
119672024-01-10T18:00:04.189Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.192673589Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603}
119682024-01-10T18:00:04.189Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193139249Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603}
119692024-01-10T18:00:04.194Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193210484Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603}
119702024-01-10T18:00:04.194Z{"msg":"Crucible runtime is spawned
119712024-01-10T18:00:04.194ZCrucible 776110dd-808f-4bfe-be77-d237fe4bd189 has session id: dbdff465-675d-43b0-b6fd-10a0ec1ae8c2","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193282866Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119722024-01-10T18:00:04.194Z{"msg":"Upstairs opts: Upstairs UUID: 776110dd-808f-4bfe-be77-d237fe4bd189, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193370568Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119732024-01-10T18:00:04.194Z{"msg":"Crucible stats registered with UUID: 776110dd-808f-4bfe-be77-d237fe4bd189","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193451611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119742024-01-10T18:00:04.194Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193542746Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603}
119752024-01-10T18:00:04.194Z{"msg":"776110dd-808f-4bfe-be77-d237fe4bd189 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193580597Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119762024-01-10T18:00:04.194Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193648819Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119772024-01-10T18:00:04.194Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193725277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119782024-01-10T18:00:04.195Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193798259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119792024-01-10T18:00:04.197Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193845681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119802024-01-10T18:00:04.197Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193913984Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119812024-01-10T18:00:04.197Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.193961963Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119822024-01-10T18:00:04.197Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.194032684Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119832024-01-10T18:00:04.198Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.194079282Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119842024-01-10T18:00:04.198Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.194157312Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119852024-01-10T18:00:04.198Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.194195155Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119862024-01-10T18:00:04.198Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.194237105Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119872024-01-10T18:00:04.198Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.194475737Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119882024-01-10T18:00:04.199Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.195256214Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119892024-01-10T18:00:04.199Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.195406328Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119902024-01-10T18:00:04.199Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.195613249Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119912024-01-10T18:00:04.199Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.195727755Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119922024-01-10T18:00:04.199Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.196212674Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119932024-01-10T18:00:04.199Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.196375306Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119942024-01-10T18:00:04.199Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.196652986Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119952024-01-10T18:00:04.199Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.196714136Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119962024-01-10T18:00:04.199Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.197322087Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119972024-01-10T18:00:04.199Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.197396277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119982024-01-10T18:00:04.202Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.197422217Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
119992024-01-10T18:00:04.207Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.197545314Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120002024-01-10T18:00:04.207Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.1975979Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120012024-01-10T18:00:04.207Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.19762308Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120022024-01-10T18:00:04.207Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.197950532Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120032024-01-10T18:00:04.207Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.198630421Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120042024-01-10T18:00:04.207Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.198715878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120052024-01-10T18:00:04.207Z{"msg":"[0]R flush_numbers[0..12]: [204, 204, 196, 207, 208, 196, 204, 204, 207, 207, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.198830732Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120062024-01-10T18:00:04.207Z{"msg":"[0]R generation[0..12]: [107, 107, 102, 109, 110, 102, 107, 107, 109, 109, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.198888364Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120072024-01-10T18:00:04.207Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.198915029Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120082024-01-10T18:00:04.207Z{"msg":"[1]R flush_numbers[0..12]: [204, 204, 196, 207, 208, 196, 204, 204, 207, 207, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.198940303Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120092024-01-10T18:00:04.207Z{"msg":"[1]R generation[0..12]: [107, 107, 102, 109, 110, 102, 107, 107, 109, 109, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.198998332Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120102024-01-10T18:00:04.207Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199025879Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120112024-01-10T18:00:04.207Z{"msg":"[2]R flush_numbers[0..12]: [204, 204, 196, 207, 208, 196, 204, 204, 207, 207, 209, 210]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
120122024-01-10T18:00:04.207Z,"time":"2024-01-10T18:00:04.199100878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120132024-01-10T18:00:04.207Z{"msg":"[2]R generation[0..12]: [107, 107, 102, 109, 110, 102, 107, 107, 109, 109, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199129419Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120142024-01-10T18:00:04.207Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199159186Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120152024-01-10T18:00:04.207Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.19921819Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120162024-01-10T18:00:04.207Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199244473Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120172024-01-10T18:00:04.207Z{"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199298852Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120182024-01-10T18:00:04.208Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199324818Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120192024-01-10T18:00:04.208Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199380937Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120202024-01-10T18:00:04.208Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.1994047Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120212024-01-10T18:00:04.208Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199432801Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120222024-01-10T18:00:04.208Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199486495Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120232024-01-10T18:00:04.208Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199511874Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120242024-01-10T18:00:04.208Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199537524Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120252024-01-10T18:00:04.208Z{"msg":"776110dd-808f-4bfe-be77-d237fe4bd189 is now active with session: dbdff465-675d-43b0-b6fd-10a0ec1ae8c2","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199590681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120262024-01-10T18:00:04.208Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199613761Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603,"session_id":"dbdff465-675d-43b0-b6fd-10a0ec1ae8c2"}
120272024-01-10T18:00:04.208Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:04.199641909Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1603}
120282024-01-10T18:00:04.208ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
120292024-01-10T18:00:04.209ZLoading write count information from file "/tmp/test_repair_verify.data"
120302024-01-10T18:00:04.210ZRun Repair workload
120312024-01-10T18:00:04.210Z01/30 Flush
120322024-01-10T18:00:04.210Z02/30 Write block 437 len 12288 data: 5 6 8
120332024-01-10T18:00:04.210Z03/30 Read block 71 len 40960
120342024-01-10T18:00:04.210Z04/30 Read block 38 len 8192
120352024-01-10T18:00:04.210Z05/30 Write block 264 len 24576 data: 7 5 8 10 8 9
120362024-01-10T18:00:04.213Z06/30 Write block 41 len 16384 data: 7 7 6 6
120372024-01-10T18:00:04.213Z07/30 Read block 7 len 20480
120382024-01-10T18:00:04.213Z08/30 Read block 489 len 36864
120392024-01-10T18:00:04.213Z09/30 Read block 278 len 24576
120402024-01-10T18:00:04.218Z10/30 Write block 344 len 20480 data: 10 10 10 11 8
120412024-01-10T18:00:04.218Z11/30 Read block 202 len 16384
120422024-01-10T18:00:04.221Z12/30 Read block 314 len 36864
120432024-01-10T18:00:04.221Z13/30 Read block 115 len 8192
120442024-01-10T18:00:04.224Z14/30 Write block 154 len 36864 data: 8 9 9 7 5 4 6 9 11
120452024-01-10T18:00:04.224Z15/30 Read block 94 len 20480
120462024-01-10T18:00:04.228Z16/30 Write block 521 len 16384 data: 10 10 5 4
120472024-01-10T18:00:04.228Z17/30 Read block 569 len 28672
120482024-01-10T18:00:04.231Z18/30 Read block 127 len 24576
120492024-01-10T18:00:04.234Z19/30 Read block 558 len 20480
120502024-01-10T18:00:04.237Z20/30 Read block 264 len 16384
120512024-01-10T18:00:04.237Z21/30 Flush
120522024-01-10T18:00:04.241Z22/30 Write block 292 len 24576 data: 13 10 9 8 8 12
120532024-01-10T18:00:04.244Z23/30 Write block 349 len 36864 data: 7 6 6 7 7 8 6 7 8
120542024-01-10T18:00:04.244Z24/30 Read block 430 len 4096
120552024-01-10T18:00:04.248Z25/30 Read block 394 len 24576
120562024-01-10T18:00:04.249Z26/30 Read block 337 len 8192
120572024-01-10T18:00:04.253Z27/30 Write block 66 len 40960 data: 11 9 8 7 8 5 4 5 7 10
120582024-01-10T18:00:04.253Z28/30 Read block 483 len 20480
120592024-01-10T18:00:04.253Z29/30 Read block 396 len 20480
120602024-01-10T18:00:04.259Z30/30 Read block 480 len 16384
120612024-01-10T18:00:04.259Z----------------------------------------------------------------
120622024-01-10T18:00:04.259Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
120632024-01-10T18:00:04.259ZStates: Active Active Active
120642024-01-10T18:00:04.259ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
120652024-01-10T18:00:04.259Z 1 Acked 1000 Flush 0 Done Sent Done false
120662024-01-10T18:00:04.259Z 2 Acked 1001 Write 3 Done Sent Done false
120672024-01-10T18:00:04.259Z 3 Acked 1002 Read 10 Done Sent Done false
120682024-01-10T18:00:04.259Z 4 Acked 1003 Read 2 Done Sent Done false
120692024-01-10T18:00:04.259Z 5 Acked 1004 Write 6 Done Sent Done false
120702024-01-10T18:00:04.259Z 6 Acked 1005 Write 4 Done Sent Done false
120712024-01-10T18:00:04.259Z 7 Acked 1006 Read 5 Done Sent Done false
120722024-01-10T18:00:04.259Z 8 Acked 1007 Read 9 Done Sent Done false
120732024-01-10T18:00:04.259Z 9 Acked 1008 Read 6 Done Sent Done false
120742024-01-10T18:00:04.259Z 10 Acked 1009 Write 5 Done Sent Done false
120752024-01-10T18:00:04.259Z 11 Acked 1010 Read 4 Done Sent Done false
120762024-01-10T18:00:04.260Z 12 Acked 1011 Read 9 Done Sent Done false
120772024-01-10T18:00:04.260Z 13 Acked 1012 Read 2 Done Sent Done false
120782024-01-10T18:00:04.261Z 14 Acked 1013 Write 9 Done Sent Done false
120792024-01-10T18:00:04.261Z 15 Acked 1014 Read 5 Done Sent Done false
120802024-01-10T18:00:04.261Z 16 Acked 1015 Write 4 Done Sent Done false
120812024-01-10T18:00:04.262Z 17 Acked 1016 Read 7 Done Sent Done false
120822024-01-10T18:00:04.262Z 18 Acked 1017 Read 6 Done Sent Done false
120832024-01-10T18:00:04.262Z 19 Acked 1018 Read 5 Done Sent Done false
120842024-01-10T18:00:04.263Z 20 Acked 1019 Read 4 Done Sent Done false
120852024-01-10T18:00:04.263Z 21 Acked 1020 Flush 0 Done Sent Done false
120862024-01-10T18:00:04.263Z 22 Acked 1021 Write 6 Done Sent Done false
120872024-01-10T18:00:04.263Z 23 Acked 1022 Write 9 Done Sent Done false
120882024-01-10T18:00:04.263Z 24 Acked 1023 Read 1 Done Sent Done false
120892024-01-10T18:00:04.263Z 25 Acked 1024 Read 6 Done Sent Done false
120902024-01-10T18:00:04.263Z 26 Acked 1025 Read 2 Done Sent Done false
120912024-01-10T18:00:04.263Z 27 Acked 1026 Write 10 Done Sent Done false
120922024-01-10T18:00:04.263Z 28 Acked 1027 Read 5 Done Sent Done false
120932024-01-10T18:00:04.263Z 29 Acked 1028 Read 5 Done Sent Done false
120942024-01-10T18:00:04.263Z 30 Acked 1029 Read 4 Done Sent Sent false
120952024-01-10T18:00:04.263Z STATES DS:0 DS:1 DS:2 TOTAL
120962024-01-10T18:00:04.264Z New 0 0 0 0
120972024-01-10T18:00:04.264Z Sent 0 30 1 31
120982024-01-10T18:00:04.264Z Done 30 0 29 59
120992024-01-10T18:00:04.264Z Skipped 0 0 0 0
121002024-01-10T18:00:04.264Z Error 0 0 0 0
121012024-01-10T18:00:04.264ZLast Flush: 1020 0 1020
121022024-01-10T18:00:04.264ZDownstairs last five completed:
121032024-01-10T18:00:04.264ZUpstairs last five completed: 30 29 28 27 26
121042024-01-10T18:00:04.269ZWrote out file "/tmp/test_repair_verify.data"
121052024-01-10T18:00:04.269Z
121062024-01-10T18:00:04.269ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121072024-01-10T18:00:04.291ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121082024-01-10T18:00:04.291Z 0 000-019 107 107 107 204 204 204 F F F
121092024-01-10T18:00:04.291Z 1 020-039 107 107 107 204 204 204 F F F
121102024-01-10T18:00:04.291Z 2 040-059 112 102 112 212 196 212 F F F <---
121112024-01-10T18:00:04.291Z 3 060-079 109 109 109 207 207 207 T F T <---
121122024-01-10T18:00:04.291Z 4 080-099 110 110 110 208 208 208 F F F
121132024-01-10T18:00:04.295Z 5 100-119 102 102 102 196 196 196 F F F
121142024-01-10T18:00:04.295Z 6 120-139 107 107 107 204 204 204 F F F
121152024-01-10T18:00:04.295Z 7 140-159 112 107 112 212 204 212 F F F <---
121162024-01-10T18:00:04.295Z 8 160-179 112 109 112 212 207 212 F F F <---
121172024-01-10T18:00:04.295Z 9 180-199 109 109 109 207 207 207 F F F
121182024-01-10T18:00:04.295Z 10 200-219 110 110 110 209 209 209 F F F
121192024-01-10T18:00:04.295Z 11 220-239 111 111 111 210 210 210 F F F
121202024-01-10T18:00:04.295Z 12 240-259 111 111 111 210 210 210 F F F
121212024-01-10T18:00:04.295Z 13 260-279 112 110 112 212 209 212 F F F <---
121222024-01-10T18:00:04.295Z 14 280-299 106 106 106 203 203 203 T F T <---
121232024-01-10T18:00:04.295Z 15 300-319 110 110 110 209 209 209 F F F
121242024-01-10T18:00:04.295Z 16 320-339 110 110 110 209 209 209 F F F
121252024-01-10T18:00:04.295Z 17 340-359 112 110 112 212 209 212 T F T <---
121262024-01-10T18:00:04.296Z 18 360-379 107 107 107 204 204 204 F F F
121272024-01-10T18:00:04.296Z 19 380-399 96 96 96 184 184 184 F F F
121282024-01-10T18:00:04.296Z 20 400-419 110 110 110 209 209 209 F F F
121292024-01-10T18:00:04.296Z 21 420-439 112 110 112 212 209 212 F F F <---
121302024-01-10T18:00:04.296Z 22 440-459 111 111 111 210 210 210 F F F
121312024-01-10T18:00:04.296Z 23 460-479 111 111 111 210 210 210 F F F
121322024-01-10T18:00:04.296Z 24 480-499 107 107 107 204 204 204 F F F
121332024-01-10T18:00:04.296Z 25 500-519 110 110 110 209 209 209 F F F
121342024-01-10T18:00:04.296Z 26 520-539 112 111 112 212 210 212 F F F <---
121352024-01-10T18:00:04.296Z 27 540-559 111 111 111 210 210 210 F F F
121362024-01-10T18:00:04.296Z 28 560-579 110 110 110 209 209 209 F F F
121372024-01-10T18:00:04.297Z 29 580-599 110 110 110 209 209 209 F F F
121382024-01-10T18:00:04.297ZMax gen: 112, Max flush: 212
121392024-01-10T18:00:04.297ZError: Difference in extent metadata found!
121402024-01-10T18:00:04.297ZA Difference in extent metadata is expected here
121412024-01-10T18:00:04.297ZOn loop 56
121422024-01-10T18:00:04.297Z
121432024-01-10T18:00:04.300ZVerifying data now
121442024-01-10T18:00:09.255ZNo change after verify
121452024-01-10T18:00:09.255ZLoop: 56 Downstairs dump after verify (and repair):
121462024-01-10T18:00:09.278ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121472024-01-10T18:00:09.278Z 0 000-019 107 107 107 204 204 204 F F F
121482024-01-10T18:00:09.278Z 1 020-039 107 107 107 204 204 204 F F F
121492024-01-10T18:00:09.278Z 2 040-059 112 112 112 212 212 212 F F F
121502024-01-10T18:00:09.279Z 3 060-079 113 113 113 213 213 213 F F F
121512024-01-10T18:00:09.279Z 4 080-099 110 110 110 208 208 208 F F F
121522024-01-10T18:00:09.279Z 5 100-119 102 102 102 196 196 196 F F F
121532024-01-10T18:00:09.280Z 6 120-139 107 107 107 204 204 204 F F F
121542024-01-10T18:00:09.280Z 7 140-159 112 112 112 212 212 212 F F F
121552024-01-10T18:00:09.288Z 8 160-179 112 112 112 212 212 212 F F F
121562024-01-10T18:00:09.288Z 9 180-199 109 109 109 207 207 207 F F F
121572024-01-10T18:00:09.288Z 10 200-219 110 110 110 209 209 209 F F F
121582024-01-10T18:00:09.288Z 11 220-239 111 111 111 210 210 210 F F F
121592024-01-10T18:00:09.288Z 12 240-259 111 111 111 210 210 210 F F F
121602024-01-10T18:00:09.288Z 13 260-279 112 112 112 212 212 212 F F F
121612024-01-10T18:00:09.288Z 14 280-299 113 113 113 213 213 213 F F F
121622024-01-10T18:00:09.288Z 15 300-319 110 110 110 209 209 209 F F F
121632024-01-10T18:00:09.288Z 16 320-339 110 110 110 209 209 209 F F F
121642024-01-10T18:00:09.288Z 17 340-359 113 113 113 213 213 213 F F F
121652024-01-10T18:00:09.288Z 18 360-379 107 107 107 204 204 204 F F F
121662024-01-10T18:00:09.288Z 19 380-399 96 96 96 184 184 184 F F F
121672024-01-10T18:00:09.288Z 20 400-419 110 110 110 209 209 209 F F F
121682024-01-10T18:00:09.288Z 21 420-439 112 112 112 212 212 212 F F F
121692024-01-10T18:00:09.288Z 22 440-459 111 111 111 210 210 210 F F F
121702024-01-10T18:00:09.288Z 23 460-479 111 111 111 210 210 210 F F F
121712024-01-10T18:00:09.288Z 24 480-499 107 107 107 204 204 204 F F F
121722024-01-10T18:00:09.288Z 25 500-519 110 110 110 209 209 209 F F F
121732024-01-10T18:00:09.289Z 26 520-539 112 112 112 212 212 212 F F F
121742024-01-10T18:00:09.289Z 27 540-559 111 111 111 210 210 210 F F F
121752024-01-10T18:00:09.289Z 28 560-579 110 110 110 209 209 209 F F F
121762024-01-10T18:00:09.289Z 29 580-599 110 110 110 209 209 209 F F F
121772024-01-10T18:00:09.289ZMax gen: 113, Max flush: 213
121782024-01-10T18:00:09.289Z
121792024-01-10T18:00:09.289ZBegin loop 57
121802024-01-10T18:00:09.289ZDownstairs to restart: 0
121812024-01-10T18:00:09.310Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.314698631Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611}
121822024-01-10T18:00:09.315Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
121832024-01-10T18:00:09.315Z,"time":"2024-01-10T18:00:09.315469969Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611}
121842024-01-10T18:00:09.315Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.315780275Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611}
121852024-01-10T18:00:09.315Z{"msg":"Crucible 90fe8014-72c7-48e6-91d5-eae3bc63f40a has session id: d7d04b29-2a4a-47a1-be87-32ddab5070ee","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.315882755Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121862024-01-10T18:00:09.315Z{"msg":"Upstairs opts: Upstairs UUID: 90fe8014-72c7-48e6-91d5-eae3bc63f40a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.315946814Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121872024-01-10T18:00:09.315Z{"msg":"Crucible stats registered with UUID: 90fe8014-72c7-48e6-91d5-eae3bc63f40a","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.315990357Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121882024-01-10T18:00:09.315Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.31605817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611}
121892024-01-10T18:00:09.317Z{"msg":"90fe8014-72c7-48e6-91d5-eae3bc63f40a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316213002Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121902024-01-10T18:00:09.317Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316284428Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121912024-01-10T18:00:09.317Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316312262Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121922024-01-10T18:00:09.317Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316373258Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121932024-01-10T18:00:09.317Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316430802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121942024-01-10T18:00:09.317Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316457105Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121952024-01-10T18:00:09.317Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316523825Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121962024-01-10T18:00:09.317Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316570033Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121972024-01-10T18:00:09.317Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316677323Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121982024-01-10T18:00:09.317Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316755112Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
121992024-01-10T18:00:09.317Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316793917Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122002024-01-10T18:00:09.317Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316817669Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122012024-01-10T18:00:09.317Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.316839994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122022024-01-10T18:00:09.317Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.318068239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122032024-01-10T18:00:09.317Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.318146446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122042024-01-10T18:00:09.317Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.318274791Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122052024-01-10T18:00:09.317Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.318366607Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122062024-01-10T18:00:09.317Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.318396799Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122072024-01-10T18:00:09.317Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.318419714Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122082024-01-10T18:00:09.322Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.319170527Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122092024-01-10T18:00:09.322Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.319438625Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122102024-01-10T18:00:09.324Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.31955331Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122112024-01-10T18:00:09.325Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320306525Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122122024-01-10T18:00:09.325Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.32035536Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122132024-01-10T18:00:09.325Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320381219Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122142024-01-10T18:00:09.325Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320402116Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122152024-01-10T18:00:09.325Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320424509Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122162024-01-10T18:00:09.325Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.32050744Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122172024-01-10T18:00:09.325Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320532574Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122182024-01-10T18:00:09.325Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.32055489Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122192024-01-10T18:00:09.325Z{"msg":"[0]R flush_numbers[0..12]: [204, 204, 212, 213, 208, 196, 204, 212, 212, 207, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320578099Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122202024-01-10T18:00:09.326Z{"msg":"[0]R generation[0..12]: [107, 107, 112, 113, 110, 102, 107, 112, 112, 109, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320635295Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122212024-01-10T18:00:09.326Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320660991Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122222024-01-10T18:00:09.326Z{"msg":"[1]R flush_numbers[0..12]: [204, 204, 212, 213, 208, 196, 204, 212, 212, 207, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320685663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122232024-01-10T18:00:09.326Z{"msg":"[1]R generation[0..12]: [107, 107, 112, 113, 110, 102, 107, 112, 112, 109, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320726947Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122242024-01-10T18:00:09.326Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320752436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122252024-01-10T18:00:09.326Z{"msg":"[2]R flush_numbers[0..12]: [204, 204, 212, 213, 208, 196, 204, 212, 212, 207, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320775966Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122262024-01-10T18:00:09.326Z{"msg":"Wait for a query_work_queue command to finish before sending IO
122272024-01-10T18:00:09.326Z[2]R generation[0..12]: [107, 107, 112, 113, 110, 102, 107, 112, 112, 109, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.32081969Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122282024-01-10T18:00:09.326Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320845121Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122292024-01-10T18:00:09.327Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320866543Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122302024-01-10T18:00:09.327Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320888238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122312024-01-10T18:00:09.327Z{"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320909455Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122322024-01-10T18:00:09.327Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320930663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122332024-01-10T18:00:09.327Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320952131Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122342024-01-10T18:00:09.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.320976898Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122352024-01-10T18:00:09.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.321020472Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122362024-01-10T18:00:09.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.321044402Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122372024-01-10T18:00:09.327Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.321066469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122382024-01-10T18:00:09.327Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.321087019Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122392024-01-10T18:00:09.330Z{"msg":"90fe8014-72c7-48e6-91d5-eae3bc63f40a is now active with session: d7d04b29-2a4a-47a1-be87-32ddab5070ee","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.321108754Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122402024-01-10T18:00:09.330Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.321128053Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611,"session_id":"d7d04b29-2a4a-47a1-be87-32ddab5070ee"}
122412024-01-10T18:00:09.330Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:09.321195616Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1611}
122422024-01-10T18:00:09.330ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
122432024-01-10T18:00:09.330ZLoading write count information from file "/tmp/test_repair_verify.data"
122442024-01-10T18:00:09.330ZRun Repair workload
122452024-01-10T18:00:09.330Z01/30 Write block 161 len 40960 data: 10 12 11 13 14 14 12 11 11 9
122462024-01-10T18:00:09.330Z02/30 Read block 327 len 16384
122472024-01-10T18:00:09.334Z03/30 Write block 424 len 28672 data: 10 8 7 6 7 7 5
122482024-01-10T18:00:09.334Z04/30 Read block 594 len 12288
122492024-01-10T18:00:09.334Z05/30 Read block 136 len 32768
122502024-01-10T18:00:09.334Z06/30 Write block 580 len 28672 data: 8 9 9 10 9 8 10
122512024-01-10T18:00:09.334Z07/30 Write block 491 len 40960 data: 11 14 15 15 18 17 15 15 15 12
122522024-01-10T18:00:09.334Z08/30 Write block 465 len 28672 data: 6 9 10 10 6 6 3
122532024-01-10T18:00:09.334Z09/30 Read block 154 len 32768
122542024-01-10T18:00:09.337Z10/30 Read block 151 len 40960
122552024-01-10T18:00:09.341Z11/30 Read block 45 len 28672
122562024-01-10T18:00:09.344Z12/30 Read block 301 len 16384
122572024-01-10T18:00:09.347Z13/30 Read block 60 len 36864
122582024-01-10T18:00:09.350Z14/30 Read block 12 len 4096
122592024-01-10T18:00:09.351Z15/30 Read block 245 len 40960
122602024-01-10T18:00:09.354Z16/30 Write block 587 len 32768 data: 9 8 9 8 8 7 8 9
122612024-01-10T18:00:09.355Z17/30 Read block 154 len 16384
122622024-01-10T18:00:09.358Z18/30 Read block 4 len 32768
122632024-01-10T18:00:09.361Z19/30 Write block 522 len 24576 data: 11 6 5 7 10 9
122642024-01-10T18:00:09.361Z20/30 Read block 406 len 12288
122652024-01-10T18:00:09.364Z21/30 Read block 498 len 36864
122662024-01-10T18:00:09.367Z22/30 Read block 181 len 8192
122672024-01-10T18:00:09.368Z23/30 Write block 456 len 12288 data: 9 10 7
122682024-01-10T18:00:09.368Z24/30 Read block 545 len 40960
122692024-01-10T18:00:09.371Z25/30 Write block 101 len 12288 data: 7 7 6
122702024-01-10T18:00:09.374Z26/30 Write block 249 len 24576 data: 7 6 6 7 7 5
122712024-01-10T18:00:09.374Z27/30 Write block 284 len 12288 data: 9 10 8
122722024-01-10T18:00:09.374Z28/30 Write block 500 len 8192 data: 13 12
122732024-01-10T18:00:09.378Z29/30 Read block 9 len 12288
122742024-01-10T18:00:09.378Z30/30 Write block 42 len 20480 data: 8 7 7 8 7
122752024-01-10T18:00:09.378Z----------------------------------------------------------------
122762024-01-10T18:00:09.378Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
122772024-01-10T18:00:09.378ZStates: Active Active Active
122782024-01-10T18:00:09.378ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
122792024-01-10T18:00:09.378Z 1 Acked 1000 Write 10 Done Done Done false
122802024-01-10T18:00:09.378Z 2 Acked 1001 Read 4 Done Done Done false
122812024-01-10T18:00:09.378Z 3 Acked 1002 Write 7 Done Done Done false
122822024-01-10T18:00:09.378Z 4 Acked 1003 Read 3 Done Done Done false
122832024-01-10T18:00:09.379Z 5 Acked 1004 Read 8 Sent Done Done false
122842024-01-10T18:00:09.379Z 6 Acked 1005 Write 7 Sent Done Done false
122852024-01-10T18:00:09.379Z 7 Acked 1006 Write 10 Sent Done Done false
122862024-01-10T18:00:09.379Z 8 Acked 1007 Write 7 Sent Done Done false
122872024-01-10T18:00:09.379Z 9 Acked 1008 Read 8 Sent Done Done false
122882024-01-10T18:00:09.379Z 10 Acked 1009 Read 10 Sent Done Done false
122892024-01-10T18:00:09.379Z 11 Acked 1010 Read 7 Sent Done Done false
122902024-01-10T18:00:09.379Z 12 Acked 1011 Read 4 Sent Done Done false
122912024-01-10T18:00:09.379Z 13 Acked 1012 Read 9 Sent Done Done false
122922024-01-10T18:00:09.379Z 14 Acked 1013 Read 1 Sent Done Done false
122932024-01-10T18:00:09.379Z 15 Acked 1014 Read 10 Sent Done Done false
122942024-01-10T18:00:09.380Z 16 Acked 1015 Write 8 Sent Done Done false
122952024-01-10T18:00:09.380Z 17 Acked 1016 Read 4 Sent Done Done false
122962024-01-10T18:00:09.380Z 18 Acked 1017 Read 8 Sent Done Done false
122972024-01-10T18:00:09.380Z 19 Acked 1018 Write 6 Sent Done Done false
122982024-01-10T18:00:09.380Z 20 Acked 1019 Read 3 Sent Done Done false
122992024-01-10T18:00:09.380Z 21 Acked 1020 Read 9 Sent Done Done false
123002024-01-10T18:00:09.380Z 22 Acked 1021 Read 2 Sent Done Done false
123012024-01-10T18:00:09.380Z 23 Acked 1022 Write 3 Sent Done Done false
123022024-01-10T18:00:09.380Z 24 Acked 1023 Read 10 Sent Done Done false
123032024-01-10T18:00:09.380Z 25 Acked 1024 Write 3 Sent Done Done false
123042024-01-10T18:00:09.380Z 26 Acked 1025 Write 6 Sent Done Done false
123052024-01-10T18:00:09.380Z 27 Acked 1026 Write 3 Sent Done Done false
123062024-01-10T18:00:09.380Z 28 Acked 1027 Write 2 Sent Done Done false
123072024-01-10T18:00:09.380Z 29 Acked 1028 Read 3 Sent Done Done false
123082024-01-10T18:00:09.380Z 30 Acked 1029 Write 5 Sent Sent Sent false
123092024-01-10T18:00:09.380Z STATES DS:0 DS:1 DS:2 TOTAL
123102024-01-10T18:00:09.380Z New 0 0 0 0
123112024-01-10T18:00:09.380Z Sent 26 1 1 28
123122024-01-10T18:00:09.380Z Done 4 29 29 62
123132024-01-10T18:00:09.381Z Skipped 0 0 0 0
123142024-01-10T18:00:09.381Z Error 0 0 0 0
123152024-01-10T18:00:09.381ZLast Flush: 0 0 0
123162024-01-10T18:00:09.381ZDownstairs last five completed:
123172024-01-10T18:00:09.381ZUpstairs last five completed: 30 29 28 27 26
123182024-01-10T18:00:09.388ZWrote out file "/tmp/test_repair_verify.data"
123192024-01-10T18:00:09.388Z
123202024-01-10T18:00:09.392ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123212024-01-10T18:00:09.418ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123222024-01-10T18:00:09.418Z 0 000-019 107 107 107 204 204 204 F F F
123232024-01-10T18:00:09.418Z 1 020-039 107 107 107 204 204 204 F F F
123242024-01-10T18:00:09.418Z 2 040-059 112 112 112 212 212 212 F T T <---
123252024-01-10T18:00:09.418Z 3 060-079 113 113 113 213 213 213 F F F
123262024-01-10T18:00:09.418Z 4 080-099 110 110 110 208 208 208 F F F
123272024-01-10T18:00:09.418Z 5 100-119 102 102 102 196 196 196 F T T <---
123282024-01-10T18:00:09.419Z 6 120-139 107 107 107 204 204 204 F F F
123292024-01-10T18:00:09.419Z 7 140-159 112 112 112 212 212 212 F F F
123302024-01-10T18:00:09.419Z 8 160-179 112 112 112 212 212 212 T T T
123312024-01-10T18:00:09.419Z 9 180-199 109 109 109 207 207 207 F F F
123322024-01-10T18:00:09.419Z 10 200-219 110 110 110 209 209 209 F F F
123332024-01-10T18:00:09.419Z 11 220-239 111 111 111 210 210 210 F F F
123342024-01-10T18:00:09.419Z 12 240-259 111 111 111 210 210 210 F T T <---
123352024-01-10T18:00:09.423Z 13 260-279 112 112 112 212 212 212 F F F
123362024-01-10T18:00:09.423Z 14 280-299 113 113 113 213 213 213 F T T <---
123372024-01-10T18:00:09.423Z 15 300-319 110 110 110 209 209 209 F F F
123382024-01-10T18:00:09.423Z 16 320-339 110 110 110 209 209 209 F F F
123392024-01-10T18:00:09.423Z 17 340-359 113 113 113 213 213 213 F F F
123402024-01-10T18:00:09.423Z 18 360-379 107 107 107 204 204 204 F F F
123412024-01-10T18:00:09.423Z 19 380-399 96 96 96 184 184 184 F F F
123422024-01-10T18:00:09.423Z 20 400-419 110 110 110 209 209 209 F F F
123432024-01-10T18:00:09.423Z 21 420-439 112 112 112 212 212 212 T T T
123442024-01-10T18:00:09.423Z 22 440-459 111 111 111 210 210 210 F T T <---
123452024-01-10T18:00:09.423Z 23 460-479 111 111 111 210 210 210 F T T <---
123462024-01-10T18:00:09.423Z 24 480-499 107 107 107 204 204 204 F T T <---
123472024-01-10T18:00:09.423Z 25 500-519 110 110 110 209 209 209 F T T <---
123482024-01-10T18:00:09.423Z 26 520-539 112 112 112 212 212 212 F T T <---
123492024-01-10T18:00:09.423Z 27 540-559 111 111 111 210 210 210 F F F
123502024-01-10T18:00:09.423Z 28 560-579 110 110 110 209 209 209 F F F
123512024-01-10T18:00:09.423Z 29 580-599 110 110 110 209 209 209 F T T <---
123522024-01-10T18:00:09.423ZMax gen: 113, Max flush: 213
123532024-01-10T18:00:09.423ZError: Difference in extent metadata found!
123542024-01-10T18:00:09.423ZA Difference in extent metadata is expected here
123552024-01-10T18:00:09.424ZOn loop 57
123562024-01-10T18:00:09.424Z
123572024-01-10T18:00:09.428ZVerifying data now
123582024-01-10T18:00:14.646ZNo change after verify
123592024-01-10T18:00:14.646ZLoop: 57 Downstairs dump after verify (and repair):
123602024-01-10T18:00:14.668ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123612024-01-10T18:00:14.669Z 0 000-019 107 107 107 204 204 204 F F F
123622024-01-10T18:00:14.669Z 1 020-039 107 107 107 204 204 204 F F F
123632024-01-10T18:00:14.669Z 2 040-059 114 114 114 214 214 214 F F F
123642024-01-10T18:00:14.669Z 3 060-079 113 113 113 213 213 213 F F F
123652024-01-10T18:00:14.669Z 4 080-099 110 110 110 208 208 208 F F F
123662024-01-10T18:00:14.669Z 5 100-119 114 114 114 214 214 214 F F F
123672024-01-10T18:00:14.669Z 6 120-139 107 107 107 204 204 204 F F F
123682024-01-10T18:00:14.672Z 7 140-159 112 112 112 212 212 212 F F F
123692024-01-10T18:00:14.672Z 8 160-179 114 114 114 214 214 214 F F F
123702024-01-10T18:00:14.672Z 9 180-199 109 109 109 207 207 207 F F F
123712024-01-10T18:00:14.672Z 10 200-219 110 110 110 209 209 209 F F F
123722024-01-10T18:00:14.672Z 11 220-239 111 111 111 210 210 210 F F F
123732024-01-10T18:00:14.673Z 12 240-259 114 114 114 214 214 214 F F F
123742024-01-10T18:00:14.675Z 13 260-279 112 112 112 212 212 212 F F F
123752024-01-10T18:00:14.676Z 14 280-299 114 114 114 214 214 214 F F F
123762024-01-10T18:00:14.676Z 15 300-319 110 110 110 209 209 209 F F F
123772024-01-10T18:00:14.676Z 16 320-339 110 110 110 209 209 209 F F F
123782024-01-10T18:00:14.676Z 17 340-359 113 113 113 213 213 213 F F F
123792024-01-10T18:00:14.676Z 18 360-379 107 107 107 204 204 204 F F F
123802024-01-10T18:00:14.676Z 19 380-399 96 96 96 184 184 184 F F F
123812024-01-10T18:00:14.676Z 20 400-419 110 110 110 209 209 209 F F F
123822024-01-10T18:00:14.676Z 21 420-439 114 114 114 214 214 214 F F F
123832024-01-10T18:00:14.676Z 22 440-459 114 114 114 214 214 214 F F F
123842024-01-10T18:00:14.676Z 23 460-479 114 114 114 214 214 214 F F F
123852024-01-10T18:00:14.677Z 24 480-499 114 114 114 214 214 214 F F F
123862024-01-10T18:00:14.677Z 25 500-519 114 114 114 214 214 214 F F F
123872024-01-10T18:00:14.677Z 26 520-539 114 114 114 214 214 214 F F F
123882024-01-10T18:00:14.677Z 27 540-559 111 111 111 210 210 210 F F F
123892024-01-10T18:00:14.677Z 28 560-579 110 110 110 209 209 209 F F F
123902024-01-10T18:00:14.678Z 29 580-599 114 114 114 214 214 214 F F F
123912024-01-10T18:00:14.678ZMax gen: 114, Max flush: 214
123922024-01-10T18:00:14.678Z
123932024-01-10T18:00:14.678ZBegin loop 58
123942024-01-10T18:00:14.678ZDownstairs to restart: 1
123952024-01-10T18:00:14.702Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.706055028Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619}
123962024-01-10T18:00:14.707Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.706719089Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619}
123972024-01-10T18:00:14.707Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
123982024-01-10T18:00:14.707Z,"time":"2024-01-10T18:00:14.706907096Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619}
123992024-01-10T18:00:14.707Z{"msg":"Crucible ccb9faaa-87ac-4da6-a1be-6fb0931e9e6c has session id: 4baa1dae-a2f3-4650-9661-136d351c5740","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.706964719Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124002024-01-10T18:00:14.707Z{"msg":"Upstairs opts: Upstairs UUID: ccb9faaa-87ac-4da6-a1be-6fb0931e9e6c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.70706988Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124012024-01-10T18:00:14.707Z{"msg":"Crucible stats registered with UUID: ccb9faaa-87ac-4da6-a1be-6fb0931e9e6c","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707139209Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124022024-01-10T18:00:14.707Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707178941Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619}
124032024-01-10T18:00:14.707Z{"msg":"ccb9faaa-87ac-4da6-a1be-6fb0931e9e6c active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707212876Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124042024-01-10T18:00:14.707Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707281259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124052024-01-10T18:00:14.708Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707352777Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124062024-01-10T18:00:14.708Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707440997Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124072024-01-10T18:00:14.708Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707489361Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124082024-01-10T18:00:14.708Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707640146Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124092024-01-10T18:00:14.708Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707829307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124102024-01-10T18:00:14.708Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707879074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124112024-01-10T18:00:14.709Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.707969396Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124122024-01-10T18:00:14.709Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.708012109Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124132024-01-10T18:00:14.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.708083828Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124142024-01-10T18:00:14.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.708125027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124152024-01-10T18:00:14.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.708199819Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124162024-01-10T18:00:14.709Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.709167357Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124172024-01-10T18:00:14.709Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.709276849Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124182024-01-10T18:00:14.709Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.709324751Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124192024-01-10T18:00:14.710Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.709404012Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124202024-01-10T18:00:14.710Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.710146415Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124212024-01-10T18:00:14.710Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.710390131Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124222024-01-10T18:00:14.710Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.710831905Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124232024-01-10T18:00:14.715Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.710886488Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124242024-01-10T18:00:14.715Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.71110885Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124252024-01-10T18:00:14.715Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.711165188Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124262024-01-10T18:00:14.715Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.71123183Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124272024-01-10T18:00:14.716Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.711272202Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124282024-01-10T18:00:14.716Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.711329739Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124292024-01-10T18:00:14.716Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.711372896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124302024-01-10T18:00:14.716Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.711905073Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124312024-01-10T18:00:14.716Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.712570894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124322024-01-10T18:00:14.717Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.712655726Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124332024-01-10T18:00:14.717Z{"msg":"[0]R flush_numbers[0..12]: [204, 204, 214, 213, 208, 214, 204, 212, 214, 207, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.712697782Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124342024-01-10T18:00:14.717Z{"msg":"[0]R generation[0..12]: [107, 107, 114, 113, 110, 114, 107, 112, 114, 109, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.712743742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124352024-01-10T18:00:14.717Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.712769187Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124362024-01-10T18:00:14.717Z{"msg":"[1]R flush_numbers[0..12]: [204, 204, 214, 213, 208, 214, 204, 212, 214, 207, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.712811571Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124372024-01-10T18:00:14.717Z{"msg":"[1]R generation[0..12]: [107, 107, 114, 113, 110, 114, 107, 112, 114, 109, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.712835931Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124382024-01-10T18:00:14.717Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.712859507Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124392024-01-10T18:00:14.717Z{"msg":"[2]R flush_numbers[0..12]: [204, 204, 214, 213, 208, 214, 204, 212, 214, 207, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.712883057Z",Wait for a query_work_queue command to finish before sending IO
124402024-01-10T18:00:14.718Z"hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124412024-01-10T18:00:14.718Z{"msg":"[2]R generation[0..12]: [107, 107, 114, 113, 110, 114, 107, 112, 114, 109, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.71299811Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124422024-01-10T18:00:14.718Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.71308567Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124432024-01-10T18:00:14.718Z{"msg":"Max found gen is 115","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.713136403Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124442024-01-10T18:00:14.719Z{"msg":"Generation requested: 116 >= found:115","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.713179278Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124452024-01-10T18:00:14.719Z{"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.713246228Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124462024-01-10T18:00:14.719Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.713274981Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124472024-01-10T18:00:14.719Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.71331681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124482024-01-10T18:00:14.719Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.713338648Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124492024-01-10T18:00:14.719Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.713408118Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124502024-01-10T18:00:14.719Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.713438436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124512024-01-10T18:00:14.719Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.713495692Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124522024-01-10T18:00:14.719Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.71352767Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124532024-01-10T18:00:14.719Z{"msg":"ccb9faaa-87ac-4da6-a1be-6fb0931e9e6c is now active with session: 4baa1dae-a2f3-4650-9661-136d351c5740","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.713593407Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"}
124542024-01-10T18:00:14.719Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.71362469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619,"session_id":"4baa1dae-a2f3-4650-9661-136d351c5740"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
124552024-01-10T18:00:14.719Z}
124562024-01-10T18:00:14.720Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:14.713729021Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1619}
124572024-01-10T18:00:14.720ZLoading write count information from file "/tmp/test_repair_verify.data"
124582024-01-10T18:00:14.720ZRun Repair workload
124592024-01-10T18:00:14.720Z01/30 Write block 483 len 40960 data: 5 8 7 8 7 8 9 11 12 15
124602024-01-10T18:00:14.720Z02/30 Read block 169 len 12288
124612024-01-10T18:00:14.724Z03/30 Write block 396 len 32768 data: 8 7 7 10 8 9 8 11
124622024-01-10T18:00:14.724Z04/30 Read block 149 len 32768
124632024-01-10T18:00:14.724Z05/30 Write block 567 len 40960 data: 7 8 8 6 6 5 6 8 9 9
124642024-01-10T18:00:14.724Z06/30 Write block 295 len 36864 data: 9 9 13 14 12 10 9 9 10
124652024-01-10T18:00:14.724Z07/30 Read block 594 len 16384
124662024-01-10T18:00:14.728Z08/30 Write block 34 len 12288 data: 13 12 11
124672024-01-10T18:00:14.728Z09/30 Read block 246 len 4096
124682024-01-10T18:00:14.728Z10/30 Write block 260 len 4096 data: 7
124692024-01-10T18:00:14.736Z11/30 Read block 349 len 28672
124702024-01-10T18:00:14.736Z12/30 Write block 186 len 24576 data: 9 12 14 13 12 12
124712024-01-10T18:00:14.739Z13/30 Write block 418 len 36864 data: 9 14 17 14 12 12 11 9 8
124722024-01-10T18:00:14.739Z14/30 Read block 303 len 16384
124732024-01-10T18:00:14.740Z15/30 Read block 463 len 24576
124742024-01-10T18:00:14.740Z16/30 Read block 287 len 12288
124752024-01-10T18:00:14.746Z17/30 Flush
124762024-01-10T18:00:14.766Z18/30 Write block 576 len 20480 data: 10 8 7 8 9
124772024-01-10T18:00:14.771Z19/30 Flush
124782024-01-10T18:00:14.771Z20/30 Write block 392 len 8192 data: 8 7
124792024-01-10T18:00:14.775Z21/30 Read block 336 len 32768
124802024-01-10T18:00:14.775Z22/30 Read block 34 len 8192
124812024-01-10T18:00:14.786Z23/30 Write block 153 len 36864 data: 8 9 10 10 8 6 5 7 11
124822024-01-10T18:00:14.786Z24/30 Read block 257 len 28672
124832024-01-10T18:00:14.786Z25/30 Write block 266 len 24576 data: 9 11 9 10 10 10
124842024-01-10T18:00:14.786Z26/30 Read block 74 len 20480
124852024-01-10T18:00:14.786Z27/30 Read block 365 len 40960
124862024-01-10T18:00:14.792Z28/30 Read block 229 len 28672
124872024-01-10T18:00:14.792Z29/30 Read block 509 len 12288
124882024-01-10T18:00:14.792Z30/30 Write block 343 len 20480 data: 12 11 11 11 12
124892024-01-10T18:00:14.799Z----------------------------------------------------------------
124902024-01-10T18:00:14.799Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
124912024-01-10T18:00:14.799ZStates: Active Active Active
124922024-01-10T18:00:14.799ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
124932024-01-10T18:00:14.800Z 1 Acked 1000 Write 10 Done Done Done false
124942024-01-10T18:00:14.800Z 2 Acked 1001 Read 3 Done Done Done false
124952024-01-10T18:00:14.800Z 3 Acked 1002 Write 8 Done Done Done false
124962024-01-10T18:00:14.800Z 4 Acked 1003 Read 8 Done Done Done false
124972024-01-10T18:00:14.800Z 5 Acked 1004 Write 10 Done Done Done false
124982024-01-10T18:00:14.800Z 6 Acked 1005 Write 9 Done Done Done false
124992024-01-10T18:00:14.800Z 7 Acked 1006 Read 4 Done Done Done false
125002024-01-10T18:00:14.800Z 8 Acked 1007 Write 3 Done Done Done false
125012024-01-10T18:00:14.800Z 9 Acked 1008 Read 1 Done Sent Done false
125022024-01-10T18:00:14.800Z 10 Acked 1009 Write 1 Done Sent Done false
125032024-01-10T18:00:14.800Z 11 Acked 1010 Read 7 Done Sent Done false
125042024-01-10T18:00:14.801Z 12 Acked 1011 Write 6 Done Sent Done false
125052024-01-10T18:00:14.801Z 13 Acked 1012 Write 9 Done Sent Done false
125062024-01-10T18:00:14.801Z 14 Acked 1013 Read 4 Done Sent Done false
125072024-01-10T18:00:14.802Z 15 Acked 1014 Read 6 Done Sent Done false
125082024-01-10T18:00:14.802Z 16 Acked 1015 Read 3 Done Sent Done false
125092024-01-10T18:00:14.802Z 17 Acked 1016 Flush 0 Done Sent Done false
125102024-01-10T18:00:14.802Z 18 Acked 1017 Write 5 Done Sent Done false
125112024-01-10T18:00:14.802Z 19 Acked 1018 Flush 0 Done Sent Done false
125122024-01-10T18:00:14.802Z 20 Acked 1019 Write 2 Done Sent Done false
125132024-01-10T18:00:14.802Z 21 Acked 1020 Read 8 Done Sent Done false
125142024-01-10T18:00:14.802Z 22 Acked 1021 Read 2 Done Sent Done false
125152024-01-10T18:00:14.802Z 23 Acked 1022 Write 9 Done Sent Done false
125162024-01-10T18:00:14.802Z 24 Acked 1023 Read 7 Done Sent Done false
125172024-01-10T18:00:14.802Z 25 Acked 1024 Write 6 Done Sent Done false
125182024-01-10T18:00:14.802Z 26 Acked 1025 Read 5 Done Sent Done false
125192024-01-10T18:00:14.802Z 27 Acked 1026 Read 10 Done Sent Done false
125202024-01-10T18:00:14.802Z 28 Acked 1027 Read 7 Done Sent Done false
125212024-01-10T18:00:14.802Z 29 Acked 1028 Read 3 Done Sent Done false
125222024-01-10T18:00:14.802Z 30 Acked 1029 Write 5 Sent Sent Sent false
125232024-01-10T18:00:14.802Z STATES DS:0 DS:1 DS:2 TOTAL
125242024-01-10T18:00:14.802Z New 0 0 0 0
125252024-01-10T18:00:14.802Z Sent 1 22 1 24
125262024-01-10T18:00:14.802Z Done 29 8 29 66
125272024-01-10T18:00:14.802Z Skipped 0 0 0 0
125282024-01-10T18:00:14.802Z Error 0 0 0 0
125292024-01-10T18:00:14.802ZLast Flush: 1018 0 1018
125302024-01-10T18:00:14.803ZDownstairs last five completed:
125312024-01-10T18:00:14.803ZUpstairs last five completed: 30 29 28 27 26
125322024-01-10T18:00:14.803ZWrote out file "/tmp/test_repair_verify.data"
125332024-01-10T18:00:14.806Z
125342024-01-10T18:00:14.806ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
125352024-01-10T18:00:14.836ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125362024-01-10T18:00:14.836Z 0 000-019 107 107 107 204 204 204 F F F
125372024-01-10T18:00:14.836Z 1 020-039 116 107 116 215 204 215 F T F <---
125382024-01-10T18:00:14.836Z 2 040-059 114 114 114 214 214 214 F F F
125392024-01-10T18:00:14.836Z 3 060-079 113 113 113 213 213 213 F F F
125402024-01-10T18:00:14.836Z 4 080-099 110 110 110 208 208 208 F F F
125412024-01-10T18:00:14.836Z 5 100-119 114 114 114 214 214 214 F F F
125422024-01-10T18:00:14.836Z 6 120-139 107 107 107 204 204 204 F F F
125432024-01-10T18:00:14.836Z 7 140-159 112 112 112 212 212 212 T F T <---
125442024-01-10T18:00:14.836Z 8 160-179 114 114 114 214 214 214 T F T <---
125452024-01-10T18:00:14.836Z 9 180-199 116 109 116 215 207 215 F F F <---
125462024-01-10T18:00:14.836Z 10 200-219 110 110 110 209 209 209 F F F
125472024-01-10T18:00:14.846Z 11 220-239 111 111 111 210 210 210 F F F
125482024-01-10T18:00:14.846Z 12 240-259 114 114 114 214 214 214 F F F
125492024-01-10T18:00:14.846Z 13 260-279 116 112 116 215 212 215 T F T <---
125502024-01-10T18:00:14.847Z 14 280-299 116 114 116 215 214 215 F T F <---
125512024-01-10T18:00:14.847Z 15 300-319 116 110 116 215 209 215 F T F <---
125522024-01-10T18:00:14.847Z 16 320-339 110 110 110 209 209 209 F F F
125532024-01-10T18:00:14.847Z 17 340-359 113 113 113 213 213 213 T F T <---
125542024-01-10T18:00:14.847Z 18 360-379 107 107 107 204 204 204 F F F
125552024-01-10T18:00:14.847Z 19 380-399 116 96 116 215 184 215 T T T <---
125562024-01-10T18:00:14.847Z 20 400-419 116 110 116 215 209 215 F T F <---
125572024-01-10T18:00:14.847Z 21 420-439 116 114 116 215 214 215 F F F <---
125582024-01-10T18:00:14.847Z 22 440-459 114 114 114 214 214 214 F F F
125592024-01-10T18:00:14.847Z 23 460-479 114 114 114 214 214 214 F F F
125602024-01-10T18:00:14.847Z 24 480-499 116 114 116 215 214 215 F T F <---
125612024-01-10T18:00:14.847Z 25 500-519 114 114 114 214 214 214 F F F
125622024-01-10T18:00:14.847Z 26 520-539 114 114 114 214 214 214 F F F
125632024-01-10T18:00:14.847Z 27 540-559 111 111 111 210 210 210 F F F
125642024-01-10T18:00:14.848Z 28 560-579 116 110 116 216 209 216 F T F <---
125652024-01-10T18:00:14.848Z 29 580-599 116 114 116 216 214 216 F F F <---
125662024-01-10T18:00:14.848ZMax gen: 116, Max flush: 216
125672024-01-10T18:00:14.848ZError: Difference in extent metadata found!
125682024-01-10T18:00:14.848ZA Difference in extent metadata is expected here
125692024-01-10T18:00:14.848ZOn loop 58
125702024-01-10T18:00:14.848Z
125712024-01-10T18:00:14.858ZVerifying data now
125722024-01-10T18:00:20.182ZNo change after verify
125732024-01-10T18:00:20.182ZLoop: 58 Downstairs dump after verify (and repair):
125742024-01-10T18:00:20.204ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125752024-01-10T18:00:20.204Z 0 000-019 107 107 107 204 204 204 F F F
125762024-01-10T18:00:20.204Z 1 020-039 116 116 116 215 215 215 F F F
125772024-01-10T18:00:20.204Z 2 040-059 114 114 114 214 214 214 F F F
125782024-01-10T18:00:20.204Z 3 060-079 113 113 113 213 213 213 F F F
125792024-01-10T18:00:20.205Z 4 080-099 110 110 110 208 208 208 F F F
125802024-01-10T18:00:20.205Z 5 100-119 114 114 114 214 214 214 F F F
125812024-01-10T18:00:20.245Z 6 120-139 107 107 107 204 204 204 F F F
125822024-01-10T18:00:20.245Z 7 140-159 117 117 117 217 217 217 F F F
125832024-01-10T18:00:20.245Z 8 160-179 117 117 117 217 217 217 F F F
125842024-01-10T18:00:20.245Z 9 180-199 116 116 116 215 215 215 F F F
125852024-01-10T18:00:20.245Z 10 200-219 110 110 110 209 209 209 F F F
125862024-01-10T18:00:20.245Z 11 220-239 111 111 111 210 210 210 F F F
125872024-01-10T18:00:20.245Z 12 240-259 114 114 114 214 214 214 F F F
125882024-01-10T18:00:20.245Z 13 260-279 117 117 117 217 217 217 F F F
125892024-01-10T18:00:20.245Z 14 280-299 116 116 116 215 215 215 F F F
125902024-01-10T18:00:20.245Z 15 300-319 116 116 116 215 215 215 F F F
125912024-01-10T18:00:20.245Z 16 320-339 110 110 110 209 209 209 F F F
125922024-01-10T18:00:20.245Z 17 340-359 117 117 117 217 217 217 F F F
125932024-01-10T18:00:20.245Z 18 360-379 107 107 107 204 204 204 F F F
125942024-01-10T18:00:20.245Z 19 380-399 117 117 117 217 217 217 F F F
125952024-01-10T18:00:20.245Z 20 400-419 116 116 116 215 215 215 F F F
125962024-01-10T18:00:20.245Z 21 420-439 116 116 116 215 215 215 F F F
125972024-01-10T18:00:20.246Z 22 440-459 114 114 114 214 214 214 F F F
125982024-01-10T18:00:20.246Z 23 460-479 114 114 114 214 214 214 F F F
125992024-01-10T18:00:20.246Z 24 480-499 116 116 116 215 215 215 F F F
126002024-01-10T18:00:20.246Z 25 500-519 114 114 114 214 214 214 F F F
126012024-01-10T18:00:20.246Z 26 520-539 114 114 114 214 214 214 F F F
126022024-01-10T18:00:20.246Z 27 540-559 111 111 111 210 210 210 F F F
126032024-01-10T18:00:20.247Z 28 560-579 116 116 116 216 216 216 F F F
126042024-01-10T18:00:20.247Z 29 580-599 116 116 116 216 216 216 F F F
126052024-01-10T18:00:20.247ZMax gen: 117, Max flush: 217
126062024-01-10T18:00:20.247Z
126072024-01-10T18:00:20.247ZBegin loop 59
126082024-01-10T18:00:20.247ZDownstairs to restart: 0
126092024-01-10T18:00:20.254Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.243580956Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627}
126102024-01-10T18:00:20.254Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244096008Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627}
126112024-01-10T18:00:20.254Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244127949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627}
126122024-01-10T18:00:20.254Z{"msg":"Crucible 3f38ff01-4a8e-43bd-b1c8-839ccb290e4c has session id: 7330be4d-8e59-425a-a062-4a33b3885d3e","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244184877Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126132024-01-10T18:00:20.254Z{"msg":"Upstairs opts: Upstairs UUID: 3f38ff01-4a8e-43bd-b1c8-839ccb290e4c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244249381Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126142024-01-10T18:00:20.256Z{"msg":"Crucible stats registered with UUID: 3f38ff01-4a8e-43bd-b1c8-839ccb290e4c","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
126152024-01-10T18:00:20.256Z2024-01-10T18:00:20.244310019Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126162024-01-10T18:00:20.256Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244467728Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627}
126172024-01-10T18:00:20.256Z{"msg":"3f38ff01-4a8e-43bd-b1c8-839ccb290e4c active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244587665Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126182024-01-10T18:00:20.256Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244616228Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126192024-01-10T18:00:20.256Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244663469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126202024-01-10T18:00:20.256Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.24468755Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126212024-01-10T18:00:20.256Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244746702Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126222024-01-10T18:00:20.256Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244816389Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126232024-01-10T18:00:20.256Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.244920845Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126242024-01-10T18:00:20.256Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.245039648Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126252024-01-10T18:00:20.256Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.245140756Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126262024-01-10T18:00:20.256Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.245196353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126272024-01-10T18:00:20.256Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.245273676Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126282024-01-10T18:00:20.256Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.24532146Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126292024-01-10T18:00:20.257Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.245367201Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126302024-01-10T18:00:20.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.24663949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126312024-01-10T18:00:20.257Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.246700231Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126322024-01-10T18:00:20.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.246790494Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126332024-01-10T18:00:20.257Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.24684218Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126342024-01-10T18:00:20.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.246866663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126352024-01-10T18:00:20.257Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.246890372Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126362024-01-10T18:00:20.257Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.247814176Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126372024-01-10T18:00:20.257Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.247951565Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126382024-01-10T18:00:20.257Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.248086085Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126392024-01-10T18:00:20.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.248951164Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126402024-01-10T18:00:20.257Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249022577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126412024-01-10T18:00:20.257Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249050627Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126422024-01-10T18:00:20.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249071453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126432024-01-10T18:00:20.257Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249112591Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126442024-01-10T18:00:20.260Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249136463Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126452024-01-10T18:00:20.260Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249174891Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126462024-01-10T18:00:20.260Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249198563Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126472024-01-10T18:00:20.260Z{"msg":"[0]R flush_numbers[0..12]: [204, 215, 214, 213, 208, 214, 204, 217, 217, 215, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249239792Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126482024-01-10T18:00:20.260Z{"msg":"[0]R generation[0..12]: [107, 116, 114, 113, 110, 114, 107, 117, 117, 116, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249264555Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126492024-01-10T18:00:20.260Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.24930813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126502024-01-10T18:00:20.260Z{"msg":"[1]R flush_numbers[0..12]: [204, 215, 214, 213, 208, 214, 204, 217, 217, 215, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249333661Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126512024-01-10T18:00:20.260Z{"msg":"[1]R generation[0..12]: [107, 116, 114, 113, 110, 114, 107, 117, 117, 116, 110, 111]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
126522024-01-10T18:00:20.260Z,"time":"2024-01-10T18:00:20.249395504Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126532024-01-10T18:00:20.260Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249424095Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126542024-01-10T18:00:20.261Z{"msg":"[2]R flush_numbers[0..12]: [204, 215, 214, 213, 208, 214, 204, 217, 217, 215, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249467347Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126552024-01-10T18:00:20.261Z{"msg":"[2]R generation[0..12]: [107, 116, 114, 113, 110, 114, 107, 117, 117, 116, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249491791Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126562024-01-10T18:00:20.261Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249545702Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126572024-01-10T18:00:20.261Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249568813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126582024-01-10T18:00:20.261Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249590988Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126592024-01-10T18:00:20.261Z{"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249630593Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126602024-01-10T18:00:20.261Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249653454Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126612024-01-10T18:00:20.261Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249675855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126622024-01-10T18:00:20.261Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249715822Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126632024-01-10T18:00:20.261Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249738975Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126642024-01-10T18:00:20.262Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249784214Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126652024-01-10T18:00:20.262Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.2498074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126662024-01-10T18:00:20.262Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249850315Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126672024-01-10T18:00:20.262Z{"msg":"3f38ff01-4a8e-43bd-b1c8-839ccb290e4c is now active with session: 7330be4d-8e59-425a-a062-4a33b3885d3e","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249873355Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126682024-01-10T18:00:20.262Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.2498928Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627,"session_id":"7330be4d-8e59-425a-a062-4a33b3885d3e"}
126692024-01-10T18:00:20.262Z{"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
126702024-01-10T18:00:20.262Zmsg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:20.249936366Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1627}
126712024-01-10T18:00:20.262ZLoading write count information from file "/tmp/test_repair_verify.data"
126722024-01-10T18:00:20.262ZRun Repair workload
126732024-01-10T18:00:20.262Z01/30 Write block 92 len 24576 data: 10 10 10 13 12 12
126742024-01-10T18:00:20.262Z02/30 Flush
126752024-01-10T18:00:20.262Z03/30 Write block 156 len 16384 data: 11 9 7 6
126762024-01-10T18:00:20.266Z04/30 Read block 449 len 20480
126772024-01-10T18:00:20.266Z05/30 Read block 572 len 20480
126782024-01-10T18:00:20.266Z06/30 Read block 120 len 4096
126792024-01-10T18:00:20.266Z07/30 Read block 110 len 28672
126802024-01-10T18:00:20.266Z08/30 Read block 467 len 28672
126812024-01-10T18:00:20.267Z09/30 Write block 5 len 28672 data: 8 7 6 5 7 8 9
126822024-01-10T18:00:20.267Z10/30 Write block 505 len 40960 data: 9 11 12 7 10 10 12 13 13 15
126832024-01-10T18:00:20.267Z11/30 Read block 101 len 24576
126842024-01-10T18:00:20.271Z12/30 Read block 494 len 32768
126852024-01-10T18:00:20.274Z13/30 Read block 5 len 20480
126862024-01-10T18:00:20.277Z14/30 Read block 313 len 32768
126872024-01-10T18:00:20.277Z15/30 Write block 435 len 4096 data: 3
126882024-01-10T18:00:20.280Z16/30 Write block 525 len 20480 data: 8 11 10 10 11
126892024-01-10T18:00:20.280Z17/30 Read block 122 len 36864
126902024-01-10T18:00:20.283Z18/30 Write block 373 len 8192 data: 7 7
126912024-01-10T18:00:20.283Z19/30 Read block 514 len 12288
126922024-01-10T18:00:20.287Z20/30 Write block 83 len 16384 data: 9 7 6 6
126932024-01-10T18:00:20.287Z21/30 Read block 409 len 28672
126942024-01-10T18:00:20.291Z22/30 Write block 520 len 24576 data: 10 11 12 7 6 9
126952024-01-10T18:00:20.291Z23/30 Write block 97 len 16384 data: 13 9 10 9
126962024-01-10T18:00:20.296Z24/30 Write block 142 len 32768 data: 11 10 10 9 9 11 11 11
126972024-01-10T18:00:20.296Z25/30 Write block 458 len 32768 data: 8 7 8 7 7 8 8 7
126982024-01-10T18:00:20.296Z26/30 Write block 331 len 12288 data: 6 4 4
126992024-01-10T18:00:20.296Z27/30 Read block 257 len 32768
127002024-01-10T18:00:20.299Z28/30 Read block 248 len 8192
127012024-01-10T18:00:20.299Z29/30 Read block 257 len 28672
127022024-01-10T18:00:20.303Z30/30 Read block 361 len 32768
127032024-01-10T18:00:20.306Z----------------------------------------------------------------
127042024-01-10T18:00:20.306Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:28
127052024-01-10T18:00:20.306ZStates: Active Active Active
127062024-01-10T18:00:20.308ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127072024-01-10T18:00:20.308Z 3 Acked 1002 Write 4 Done Done Done false
127082024-01-10T18:00:20.308Z 4 Acked 1003 Read 5 Done Done Done false
127092024-01-10T18:00:20.308Z 5 Acked 1004 Read 5 Done Done Done false
127102024-01-10T18:00:20.308Z 6 Acked 1005 Read 1 Done Done Done false
127112024-01-10T18:00:20.308Z 7 Acked 1006 Read 7 Sent Done Done false
127122024-01-10T18:00:20.308Z 8 Acked 1007 Read 7 Sent Done Done false
127132024-01-10T18:00:20.308Z 9 Acked 1008 Write 7 Sent Done Done false
127142024-01-10T18:00:20.310Z 10 Acked 1009 Write 10 Sent Done Done false
127152024-01-10T18:00:20.310Z 11 Acked 1010 Read 6 Sent Done Done false
127162024-01-10T18:00:20.310Z 12 Acked 1011 Read 8 Sent Done Done false
127172024-01-10T18:00:20.310Z 13 Acked 1012 Read 5 Sent Done Done false
127182024-01-10T18:00:20.310Z 14 Acked 1013 Read 8 Sent Done Done false
127192024-01-10T18:00:20.310Z 15 Acked 1014 Write 1 Sent Done Done false
127202024-01-10T18:00:20.310Z 16 Acked 1015 Write 5 Sent Done Done false
127212024-01-10T18:00:20.310Z 17 Acked 1016 Read 9 Sent Done Done false
127222024-01-10T18:00:20.310Z 18 Acked 1017 Write 2 Sent Done Done false
127232024-01-10T18:00:20.310Z 19 Acked 1018 Read 3 Sent Done Done false
127242024-01-10T18:00:20.310Z 20 Acked 1019 Write 4 Sent Done Done false
127252024-01-10T18:00:20.310Z 21 Acked 1020 Read 7 Sent Done Done false
127262024-01-10T18:00:20.310Z 22 Acked 1021 Write 6 Sent Done Done false
127272024-01-10T18:00:20.310Z 23 Acked 1022 Write 4 Sent Done Done false
127282024-01-10T18:00:20.310Z 24 Acked 1023 Write 8 Sent Done Done false
127292024-01-10T18:00:20.310Z 25 Acked 1024 Write 8 Sent Done Done false
127302024-01-10T18:00:20.310Z 26 Acked 1025 Write 3 Sent Done Done false
127312024-01-10T18:00:20.310Z 27 Acked 1026 Read 8 Sent Done Done false
127322024-01-10T18:00:20.310Z 28 Acked 1027 Read 2 Sent Done Done false
127332024-01-10T18:00:20.310Z 29 Acked 1028 Read 7 Sent Done Done false
127342024-01-10T18:00:20.310Z 30 Acked 1029 Read 8 Sent Done Sent false
127352024-01-10T18:00:20.310Z STATES DS:0 DS:1 DS:2 TOTAL
127362024-01-10T18:00:20.310Z New 0 0 0 0
127372024-01-10T18:00:20.310Z Sent 24 0 1 25
127382024-01-10T18:00:20.311Z Done 4 28 27 59
127392024-01-10T18:00:20.311Z Skipped 0 0 0 0
127402024-01-10T18:00:20.311Z Error 0 0 0 0
127412024-01-10T18:00:20.311ZLast Flush: 1001 1001 1001
127422024-01-10T18:00:20.311ZDownstairs last five completed: 1001 1000
127432024-01-10T18:00:20.311ZUpstairs last five completed: 30 29 28 27 26
127442024-01-10T18:00:20.311ZWrote out file "/tmp/test_repair_verify.data"
127452024-01-10T18:00:20.314Z
127462024-01-10T18:00:20.318ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
127472024-01-10T18:00:20.344ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127482024-01-10T18:00:20.344Z 0 000-019 107 107 107 204 204 204 F T T <---
127492024-01-10T18:00:20.344Z 1 020-039 116 116 116 215 215 215 F F F
127502024-01-10T18:00:20.344Z 2 040-059 114 114 114 214 214 214 F F F
127512024-01-10T18:00:20.344Z 3 060-079 113 113 113 213 213 213 F F F
127522024-01-10T18:00:20.344Z 4 080-099 118 118 118 218 218 218 F T T <---
127532024-01-10T18:00:20.345Z 5 100-119 114 114 114 214 214 214 F T T <---
127542024-01-10T18:00:20.345Z 6 120-139 107 107 107 204 204 204 F F F
127552024-01-10T18:00:20.348Z 7 140-159 117 117 117 217 217 217 T T T
127562024-01-10T18:00:20.348Z 8 160-179 117 117 117 217 217 217 F F F
127572024-01-10T18:00:20.348Z 9 180-199 116 116 116 215 215 215 F F F
127582024-01-10T18:00:20.349Z 10 200-219 110 110 110 209 209 209 F F F
127592024-01-10T18:00:20.349Z 11 220-239 111 111 111 210 210 210 F F F
127602024-01-10T18:00:20.349Z 12 240-259 114 114 114 214 214 214 F F F
127612024-01-10T18:00:20.349Z 13 260-279 117 117 117 217 217 217 F F F
127622024-01-10T18:00:20.349Z 14 280-299 116 116 116 215 215 215 F F F
127632024-01-10T18:00:20.349Z 15 300-319 116 116 116 215 215 215 F F F
127642024-01-10T18:00:20.349Z 16 320-339 110 110 110 209 209 209 F T T <---
127652024-01-10T18:00:20.349Z 17 340-359 117 117 117 217 217 217 F F F
127662024-01-10T18:00:20.349Z 18 360-379 107 107 107 204 204 204 F T T <---
127672024-01-10T18:00:20.349Z 19 380-399 117 117 117 217 217 217 F F F
127682024-01-10T18:00:20.349Z 20 400-419 116 116 116 215 215 215 F F F
127692024-01-10T18:00:20.349Z 21 420-439 116 116 116 215 215 215 F T T <---
127702024-01-10T18:00:20.349Z 22 440-459 114 114 114 214 214 214 F T T <---
127712024-01-10T18:00:20.349Z 23 460-479 114 114 114 214 214 214 F T T <---
127722024-01-10T18:00:20.349Z 24 480-499 116 116 116 215 215 215 F F F
127732024-01-10T18:00:20.349Z 25 500-519 114 114 114 214 214 214 F T T <---
127742024-01-10T18:00:20.349Z 26 520-539 114 114 114 214 214 214 F T T <---
127752024-01-10T18:00:20.349Z 27 540-559 111 111 111 210 210 210 F F F
127762024-01-10T18:00:20.349Z 28 560-579 116 116 116 216 216 216 F F F
127772024-01-10T18:00:20.349Z 29 580-599 116 116 116 216 216 216 F F F
127782024-01-10T18:00:20.349ZMax gen: 118, Max flush: 218
127792024-01-10T18:00:20.349ZError: Difference in extent metadata found!
127802024-01-10T18:00:20.353ZA Difference in extent metadata is expected here
127812024-01-10T18:00:20.353ZOn loop 59
127822024-01-10T18:00:20.353Z
127832024-01-10T18:00:20.353ZVerifying data now
127842024-01-10T18:00:25.573ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
127852024-01-10T18:00:25.577Z144a145
127862024-01-10T18:00:25.577Z> 11,
127872024-01-10T18:00:25.577Z145a147
127882024-01-10T18:00:25.577Z> 10,
127892024-01-10T18:00:25.577Z148,149c150,152
127902024-01-10T18:00:25.577Z< 8,
127912024-01-10T18:00:25.577Z< 8,
127922024-01-10T18:00:25.577Z---
127932024-01-10T18:00:25.577Z> 11,
127942024-01-10T18:00:25.578Z> 11,
127952024-01-10T18:00:25.578Z> 11,
127962024-01-10T18:00:25.578Z151,153d153
127972024-01-10T18:00:25.578Z< 10,
127982024-01-10T18:00:25.578Z< 10,
127992024-01-10T18:00:25.578Z< 10,
128002024-01-10T18:00:25.578Zdiff found after verify
128012024-01-10T18:00:25.583ZLoop: 59 Downstairs dump after verify (and repair):
128022024-01-10T18:00:25.609ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128032024-01-10T18:00:25.609Z 0 000-019 119 119 119 219 219 219 F F F
128042024-01-10T18:00:25.609Z 1 020-039 116 116 116 215 215 215 F F F
128052024-01-10T18:00:25.609Z 2 040-059 114 114 114 214 214 214 F F F
128062024-01-10T18:00:25.609Z 3 060-079 113 113 113 213 213 213 F F F
128072024-01-10T18:00:25.609Z 4 080-099 119 119 119 219 219 219 F F F
128082024-01-10T18:00:25.610Z 5 100-119 119 119 119 219 219 219 F F F
128092024-01-10T18:00:25.613Z 6 120-139 107 107 107 204 204 204 F F F
128102024-01-10T18:00:25.613Z 7 140-159 119 119 119 219 219 219 F F F
128112024-01-10T18:00:25.613Z 8 160-179 117 117 117 217 217 217 F F F
128122024-01-10T18:00:25.613Z 9 180-199 116 116 116 215 215 215 F F F
128132024-01-10T18:00:25.613Z 10 200-219 110 110 110 209 209 209 F F F
128142024-01-10T18:00:25.613Z 11 220-239 111 111 111 210 210 210 F F F
128152024-01-10T18:00:25.613Z 12 240-259 114 114 114 214 214 214 F F F
128162024-01-10T18:00:25.613Z 13 260-279 117 117 117 217 217 217 F F F
128172024-01-10T18:00:25.613Z 14 280-299 116 116 116 215 215 215 F F F
128182024-01-10T18:00:25.613Z 15 300-319 116 116 116 215 215 215 F F F
128192024-01-10T18:00:25.613Z 16 320-339 119 119 119 219 219 219 F F F
128202024-01-10T18:00:25.614Z 17 340-359 117 117 117 217 217 217 F F F
128212024-01-10T18:00:25.614Z 18 360-379 119 119 119 219 219 219 F F F
128222024-01-10T18:00:25.614Z 19 380-399 117 117 117 217 217 217 F F F
128232024-01-10T18:00:25.614Z 20 400-419 116 116 116 215 215 215 F F F
128242024-01-10T18:00:25.614Z 21 420-439 119 119 119 219 219 219 F F F
128252024-01-10T18:00:25.614Z 22 440-459 119 119 119 219 219 219 F F F
128262024-01-10T18:00:25.614Z 23 460-479 119 119 119 219 219 219 F F F
128272024-01-10T18:00:25.614Z 24 480-499 116 116 116 215 215 215 F F F
128282024-01-10T18:00:25.614Z 25 500-519 119 119 119 219 219 219 F F F
128292024-01-10T18:00:25.614Z 26 520-539 119 119 119 219 219 219 F F F
128302024-01-10T18:00:25.614Z 27 540-559 111 111 111 210 210 210 F F F
128312024-01-10T18:00:25.615Z 28 560-579 116 116 116 216 216 216 F F F
128322024-01-10T18:00:25.615Z 29 580-599 116 116 116 216 216 216 F F F
128332024-01-10T18:00:25.615ZMax gen: 119, Max flush: 219
128342024-01-10T18:00:25.615Z
128352024-01-10T18:00:25.615ZBegin loop 60
128362024-01-10T18:00:25.615ZDownstairs to restart: 1
128372024-01-10T18:00:25.643Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.64626358Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638}
128382024-01-10T18:00:25.643Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.646800166Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638}
128392024-01-10T18:00:25.643Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.646864879Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638}
128402024-01-10T18:00:25.656Z{"msg":"Crucible 07f40f64-d2cf-4a2c-8ed3-1c58e5c1b72a has session id: 9b6c7375-b0ed-4b6b-958d-75fa1648b963","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.646904656Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128412024-01-10T18:00:25.656Z{"msg":"Upstairs opts: Upstairs UUID: 07f40f64-d2cf-4a2c-8ed3-1c58e5c1b72a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"Crucible runtime is spawned
128422024-01-10T18:00:25.656Zcrucible","level":30,"time":"2024-01-10T18:00:25.647017491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128432024-01-10T18:00:25.656Z{"msg":"Crucible stats registered with UUID: 07f40f64-d2cf-4a2c-8ed3-1c58e5c1b72a","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647066884Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128442024-01-10T18:00:25.656Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647123153Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638}
128452024-01-10T18:00:25.656Z{"msg":"07f40f64-d2cf-4a2c-8ed3-1c58e5c1b72a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647452016Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128462024-01-10T18:00:25.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647518559Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"0","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128472024-01-10T18:00:25.656Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647545974Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"0","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128482024-01-10T18:00:25.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647613739Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"1","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128492024-01-10T18:00:25.656Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.64765773Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"1","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128502024-01-10T18:00:25.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647699256Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"2","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128512024-01-10T18:00:25.657Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647740563Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"2","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128522024-01-10T18:00:25.657Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647771378Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128532024-01-10T18:00:25.657Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647811237Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128542024-01-10T18:00:25.658Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.647902931Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128552024-01-10T18:00:25.658Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.648053588Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128562024-01-10T18:00:25.658Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.648134388Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128572024-01-10T18:00:25.658Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.648184451Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128582024-01-10T18:00:25.658Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.649427771Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"2","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128592024-01-10T18:00:25.658Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.649487233Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"2","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128602024-01-10T18:00:25.658Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.649531365Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"0","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128612024-01-10T18:00:25.660Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.649616416Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"0","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128622024-01-10T18:00:25.660Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.649662757Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"1","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128632024-01-10T18:00:25.660Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.649736309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"1","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128642024-01-10T18:00:25.660Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.650530798Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"2","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128652024-01-10T18:00:25.660Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.650606239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"0","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128662024-01-10T18:00:25.660Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.650980165Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"1","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128672024-01-10T18:00:25.660Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651355322Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"0","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128682024-01-10T18:00:25.660Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651446278Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"0","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128692024-01-10T18:00:25.660Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651476051Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128702024-01-10T18:00:25.660Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651516399Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"2","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128712024-01-10T18:00:25.660Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651539869Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"2","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128722024-01-10T18:00:25.660Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651579673Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128732024-01-10T18:00:25.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651758196Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"1","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128742024-01-10T18:00:25.661Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651832121Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"1","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128752024-01-10T18:00:25.661Z{"msg":"[0]R flush_numbers[0..12]: [219, 215, 214, 213, 219, 219, 204, 219, 217, 215, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651904087Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128762024-01-10T18:00:25.661Z{"msg":"[0]R generation[0..12]: [119, 116, 114, 113, 119, 119, 107, 119, 117, 116, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651954803Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128772024-01-10T18:00:25.661Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.651980328Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128782024-01-10T18:00:25.661Z{"msg":"[1]R flush_numbers[0..12]: [219, 215, 214, 213, 219, 219, 204, 219, 217, 215, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652021348Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128792024-01-10T18:00:25.661Z{"msg":"[1]R generation[0..12]: [119, 116, 114, 113, 119, 119, 107, 119, 117, 116, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652045501Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128802024-01-10T18:00:25.661Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652086623Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128812024-01-10T18:00:25.661Z{"msg":"[2]R flush_numbers[0..12]: [219, 215, 214, 213, 219, 219, 204, 219, 217, 215, 209, 210]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.65211141Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128822024-01-10T18:00:25.661Z{"msg":"[2]R generation[0..12]: [119, 116, 114, 113, 119, 119, 107, 119, 117, 116, 110, 111]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652134432Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128832024-01-10T18:00:25.661Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]"Wait for a query_work_queue command to finish before sending IO
128842024-01-10T18:00:25.661Z,"v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652179648Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128852024-01-10T18:00:25.661Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652203111Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128862024-01-10T18:00:25.661Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652243392Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128872024-01-10T18:00:25.661Z{"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652265834Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128882024-01-10T18:00:25.661Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652310708Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"":"downstairs","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128892024-01-10T18:00:25.661Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652336334Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128902024-01-10T18:00:25.661Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652380235Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"0","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128912024-01-10T18:00:25.661Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652422364Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"1","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128922024-01-10T18:00:25.661Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652466598Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"client":"2","":"downstairs","session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128932024-01-10T18:00:25.661Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652490336Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128942024-01-10T18:00:25.661Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652511571Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128952024-01-10T18:00:25.661Z{"msg":"07f40f64-d2cf-4a2c-8ed3-1c58e5c1b72a is now active with session: 9b6c7375-b0ed-4b6b-958d-75fa1648b963","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652550604Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128962024-01-10T18:00:25.662Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652571315Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638,"session_id":"9b6c7375-b0ed-4b6b-958d-75fa1648b963"}
128972024-01-10T18:00:25.662Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:25.652592551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1638}
128982024-01-10T18:00:25.662ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
128992024-01-10T18:00:25.662ZLoading write count information from file "/tmp/test_repair_verify.data"
129002024-01-10T18:00:25.662ZRun Repair workload
129012024-01-10T18:00:25.674Z01/30 Write block 242 len 36864 data: 8 9 13 12 8 7 8 8 7
129022024-01-10T18:00:25.674Z02/30 Flush
129032024-01-10T18:00:25.674Z03/30 Write block 214 len 28672 data: 9 7 10 10 8 8 8
129042024-01-10T18:00:25.674Z04/30 Write block 82 len 8192 data: 8 10
129052024-01-10T18:00:25.674Z05/30 Read block 315 len 4096
129062024-01-10T18:00:25.676Z06/30 Flush
129072024-01-10T18:00:25.676Z07/30 Write block 99 len 8192 data: 11 10
129082024-01-10T18:00:25.676Z08/30 Read block 239 len 32768
129092024-01-10T18:00:25.676Z09/30 Write block 416 len 28672 data: 8 10 10 15 18 15 13
129102024-01-10T18:00:25.676Z10/30 Read block 342 len 4096
129112024-01-10T18:00:25.676Z11/30 Read block 179 len 24576
129122024-01-10T18:00:25.676Z12/30 Read block 262 len 24576
129132024-01-10T18:00:25.676Z13/30 Write block 285 len 36864 data: 11 9 8 7 10 12 13 14 11
129142024-01-10T18:00:25.676Z14/30 Read block 258 len 16384
129152024-01-10T18:00:25.682Z15/30 Read block 42 len 4096
129162024-01-10T18:00:25.682Z16/30 Write block 492 len 20480 data: 16 16 16 19 18
129172024-01-10T18:00:25.682Z17/30 Read block 292 len 36864
129182024-01-10T18:00:25.682Z18/30 Read block 569 len 28672
129192024-01-10T18:00:25.687Z19/30 Write block 395 len 12288 data: 9 9 8
129202024-01-10T18:00:25.687Z20/30 Read block 437 len 28672
129212024-01-10T18:00:25.692Z21/30 Write block 244 len 24576 data: 14 13 9 8 9 9
129222024-01-10T18:00:25.692Z22/30 Write block 546 len 4096 data: 11
129232024-01-10T18:00:25.692Z23/30 Read block 92 len 28672
129242024-01-10T18:00:26.109Z24/30 Read block 306 len 8192
129252024-01-10T18:00:26.109Z25/30 Write block 369 len 24576 data: 6 8 7 7 8 8
129262024-01-10T18:00:26.109Z26/30 Read block 409 len 24576
129272024-01-10T18:00:26.114Z27/30 Read block 229 len 4096
129282024-01-10T18:00:26.114Z28/30 Write block 112 len 32768 data: 14 14 11 9 9 9 7 8
129292024-01-10T18:00:26.114Z29/30 Read block 287 len 20480
129302024-01-10T18:00:26.114Z30/30 Read block 182 len 12288
129312024-01-10T18:00:26.114Z----------------------------------------------------------------
129322024-01-10T18:00:26.114Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
129332024-01-10T18:00:26.115ZStates: Active Active Active
129342024-01-10T18:00:26.115ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129352024-01-10T18:00:26.115Z 1 Acked 1000 Write 9 Done Sent Done false
129362024-01-10T18:00:26.115Z 2 Acked 1001 Flush 0 Done Sent Done false
129372024-01-10T18:00:26.115Z 3 Acked 1002 Write 7 Done Sent Done false
129382024-01-10T18:00:26.115Z 4 Acked 1003 Write 2 Done Sent Done false
129392024-01-10T18:00:26.115Z 5 Acked 1004 Read 1 Done Sent Done false
129402024-01-10T18:00:26.115Z 6 Acked 1005 Flush 0 Done Sent Done false
129412024-01-10T18:00:26.115Z 7 Acked 1006 Write 2 Done Sent Done false
129422024-01-10T18:00:26.115Z 8 Acked 1007 Read 8 Done Sent Done false
129432024-01-10T18:00:26.115Z 9 Acked 1008 Write 7 Done Sent Done false
129442024-01-10T18:00:26.115Z 10 Acked 1009 Read 1 Done Sent Done false
129452024-01-10T18:00:26.115Z 11 Acked 1010 Read 6 Done Sent Done false
129462024-01-10T18:00:26.115Z 12 Acked 1011 Read 6 Done Sent Done false
129472024-01-10T18:00:26.115Z 13 Acked 1012 Write 9 Done Sent Done false
129482024-01-10T18:00:26.115Z 14 Acked 1013 Read 4 Done Sent Done false
129492024-01-10T18:00:26.115Z 15 Acked 1014 Read 1 Done Sent Done false
129502024-01-10T18:00:26.115Z 16 Acked 1015 Write 5 Done Sent Done false
129512024-01-10T18:00:26.115Z 17 Acked 1016 Read 9 Done Sent Done false
129522024-01-10T18:00:26.115Z 18 Acked 1017 Read 7 Done Sent Done false
129532024-01-10T18:00:26.115Z 19 Acked 1018 Write 3 Done Sent Done false
129542024-01-10T18:00:26.119Z 20 Acked 1019 Read 7 Done Sent Done false
129552024-01-10T18:00:26.119Z 21 Acked 1020 Write 6 Done Sent Done false
129562024-01-10T18:00:26.119Z 22 Acked 1021 Write 1 Done Sent Done false
129572024-01-10T18:00:26.120Z 23 Acked 1022 Read 7 Done Sent Done false
129582024-01-10T18:00:26.120Z 24 Acked 1023 Read 2 Done Sent Done false
129592024-01-10T18:00:26.120Z 25 Acked 1024 Write 6 Done Sent Done false
129602024-01-10T18:00:26.120Z 26 Acked 1025 Read 6 Done Sent Done false
129612024-01-10T18:00:26.120Z 27 Acked 1026 Read 1 Done Sent Done false
129622024-01-10T18:00:26.120Z 28 Acked 1027 Write 8 Done Sent Done false
129632024-01-10T18:00:26.120Z 29 Acked 1028 Read 5 Done Sent Done false
129642024-01-10T18:00:26.120Z 30 Acked 1029 Read 3 Done Sent Done false
129652024-01-10T18:00:26.120Z STATES DS:0 DS:1 DS:2 TOTAL
129662024-01-10T18:00:26.120Z New 0 0 0 0
129672024-01-10T18:00:26.120Z Sent 0 30 0 30
129682024-01-10T18:00:26.120Z Done 30 0 30 60
129692024-01-10T18:00:26.120Z Skipped 0 0 0 0
129702024-01-10T18:00:26.120Z Error 0 0 0 0
129712024-01-10T18:00:26.120ZLast Flush: 1005 0 1005
129722024-01-10T18:00:26.120ZDownstairs last five completed:
129732024-01-10T18:00:26.120ZUpstairs last five completed: 30 29 28 27 26
129742024-01-10T18:00:26.120ZWrote out file "/tmp/test_repair_verify.data"
129752024-01-10T18:00:26.120Z
129762024-01-10T18:00:26.120ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
129772024-01-10T18:00:26.120ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129782024-01-10T18:00:26.122Z 0 000-019 119 119 119 219 219 219 F F F
129792024-01-10T18:00:26.122Z 1 020-039 116 116 116 215 215 215 F F F
129802024-01-10T18:00:26.122Z 2 040-059 114 114 114 214 214 214 F F F
129812024-01-10T18:00:26.122Z 3 060-079 113 113 113 213 213 213 F F F
129822024-01-10T18:00:26.122Z 4 080-099 120 119 120 221 219 221 T F T <---
129832024-01-10T18:00:26.122Z 5 100-119 119 119 119 219 219 219 T F T <---
129842024-01-10T18:00:26.122Z 6 120-139 107 107 107 204 204 204 F F F
129852024-01-10T18:00:26.122Z 7 140-159 119 119 119 219 219 219 F F F
129862024-01-10T18:00:26.122Z 8 160-179 117 117 117 217 217 217 F F F
129872024-01-10T18:00:26.122Z 9 180-199 116 116 116 215 215 215 F F F
129882024-01-10T18:00:26.122Z 10 200-219 120 110 120 221 209 221 F F F <---
129892024-01-10T18:00:26.122Z 11 220-239 120 111 120 221 210 221 F F F <---
129902024-01-10T18:00:26.122Z 12 240-259 120 114 120 220 214 220 T F T <---
129912024-01-10T18:00:26.122Z 13 260-279 117 117 117 217 217 217 F F F
129922024-01-10T18:00:26.122Z 14 280-299 116 116 116 215 215 215 T F T <---
129932024-01-10T18:00:26.122Z 15 300-319 116 116 116 215 215 215 F F F
129942024-01-10T18:00:26.122Z 16 320-339 119 119 119 219 219 219 F F F
129952024-01-10T18:00:26.122Z 17 340-359 117 117 117 217 217 217 F F F
129962024-01-10T18:00:26.122Z 18 360-379 119 119 119 219 219 219 T F T <---
129972024-01-10T18:00:26.122Z 19 380-399 117 117 117 217 217 217 T F T <---
129982024-01-10T18:00:26.123Z 20 400-419 116 116 116 215 215 215 T F T <---
129992024-01-10T18:00:26.123Z 21 420-439 119 119 119 219 219 219 T F T <---
130002024-01-10T18:00:26.123Z 22 440-459 119 119 119 219 219 219 F F F
130012024-01-10T18:00:26.123Z 23 460-479 119 119 119 219 219 219 F F F
130022024-01-10T18:00:26.124Z 24 480-499 116 116 116 215 215 215 T F T <---
130032024-01-10T18:00:26.124Z 25 500-519 119 119 119 219 219 219 F F F
130042024-01-10T18:00:26.124Z 26 520-539 119 119 119 219 219 219 F F F
130052024-01-10T18:00:26.124Z 27 540-559 111 111 111 210 210 210 T F T <---
130062024-01-10T18:00:26.124Z 28 560-579 116 116 116 216 216 216 F F F
130072024-01-10T18:00:26.124Z 29 580-599 116 116 116 216 216 216 F F F
130082024-01-10T18:00:26.124ZMax gen: 120, Max flush: 221
130092024-01-10T18:00:26.124ZError: Difference in extent metadata found!
130102024-01-10T18:00:26.124ZA Difference in extent metadata is expected here
130112024-01-10T18:00:26.124ZOn loop 60
130122024-01-10T18:00:26.124Z
130132024-01-10T18:00:26.124ZVerifying data now
130142024-01-10T18:00:30.960ZNo change after verify
130152024-01-10T18:00:30.961ZLoop: 60 Downstairs dump after verify (and repair):
130162024-01-10T18:00:30.982ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130172024-01-10T18:00:30.983Z 0 000-019 119 119 119 219 219 219 F F F
130182024-01-10T18:00:30.988Z 1 020-039 116 116 116 215 215 215 F F F
130192024-01-10T18:00:30.988Z 2 040-059 114 114 114 214 214 214 F F F
130202024-01-10T18:00:30.988Z 3 060-079 113 113 113 213 213 213 F F F
130212024-01-10T18:00:30.988Z 4 080-099 121 121 121 222 222 222 F F F
130222024-01-10T18:00:30.988Z 5 100-119 121 121 121 222 222 222 F F F
130232024-01-10T18:00:30.988Z 6 120-139 107 107 107 204 204 204 F F F
130242024-01-10T18:00:30.988Z 7 140-159 119 119 119 219 219 219 F F F
130252024-01-10T18:00:30.988Z 8 160-179 117 117 117 217 217 217 F F F
130262024-01-10T18:00:30.989Z 9 180-199 116 116 116 215 215 215 F F F
130272024-01-10T18:00:30.989Z 10 200-219 120 120 120 221 221 221 F F F
130282024-01-10T18:00:30.989Z 11 220-239 120 120 120 221 221 221 F F F
130292024-01-10T18:00:30.990Z 12 240-259 121 121 121 222 222 222 F F F
130302024-01-10T18:00:30.990Z 13 260-279 117 117 117 217 217 217 F F F
130312024-01-10T18:00:30.990Z 14 280-299 121 121 121 222 222 222 F F F
130322024-01-10T18:00:30.990Z 15 300-319 116 116 116 215 215 215 F F F
130332024-01-10T18:00:30.991Z 16 320-339 119 119 119 219 219 219 F F F
130342024-01-10T18:00:30.991Z 17 340-359 117 117 117 217 217 217 F F F
130352024-01-10T18:00:30.991Z 18 360-379 121 121 121 222 222 222 F F F
130362024-01-10T18:00:30.991Z 19 380-399 121 121 121 222 222 222 F F F
130372024-01-10T18:00:30.991Z 20 400-419 121 121 121 222 222 222 F F F
130382024-01-10T18:00:30.991Z 21 420-439 121 121 121 222 222 222 F F F
130392024-01-10T18:00:30.991Z 22 440-459 119 119 119 219 219 219 F F F
130402024-01-10T18:00:30.991Z 23 460-479 119 119 119 219 219 219 F F F
130412024-01-10T18:00:30.991Z 24 480-499 121 121 121 222 222 222 F F F
130422024-01-10T18:00:30.991Z 25 500-519 119 119 119 219 219 219 F F F
130432024-01-10T18:00:30.991Z 26 520-539 119 119 119 219 219 219 F F F
130442024-01-10T18:00:30.991Z 27 540-559 121 121 121 222 222 222 F F F
130452024-01-10T18:00:30.991Z 28 560-579 116 116 116 216 216 216 F F F
130462024-01-10T18:00:30.992Z 29 580-599 116 116 116 216 216 216 F F F
130472024-01-10T18:00:30.992ZMax gen: 121, Max flush: 222
130482024-01-10T18:00:30.992Z
130492024-01-10T18:00:30.992ZBegin loop 61
130502024-01-10T18:00:30.992ZDownstairs to restart: 1
130512024-01-10T18:00:31.017Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.021468642Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646}
130522024-01-10T18:00:31.017Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.021947991Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646}
130532024-01-10T18:00:31.018Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.021977361Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646}
130542024-01-10T18:00:31.018Z{"msg":"Crucible 9a8ded25-3d02-4984-82a4-11057a7e382e has session id: 6420e5b2-ae66-47d5-9cdc-34ac162bd4ed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.021998534Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130552024-01-10T18:00:31.026Z{"msg":"Upstairs opts: Upstairs UUID: 9a8ded25-3d02-4984-82a4-11057a7e382e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":Crucible runtime is spawned
130562024-01-10T18:00:31.026Z30,"time":"2024-01-10T18:00:31.022154228Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130572024-01-10T18:00:31.026Z{"msg":"Crucible stats registered with UUID: 9a8ded25-3d02-4984-82a4-11057a7e382e","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.022214283Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130582024-01-10T18:00:31.026Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.02233467Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646}
130592024-01-10T18:00:31.026Z{"msg":"9a8ded25-3d02-4984-82a4-11057a7e382e active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.022679269Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130602024-01-10T18:00:31.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.022778888Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"0","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130612024-01-10T18:00:31.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.022938741Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"0","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130622024-01-10T18:00:31.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.022985867Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"1","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130632024-01-10T18:00:31.027Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.023115866Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"1","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130642024-01-10T18:00:31.027Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.023406853Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"2","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130652024-01-10T18:00:31.027Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.023576676Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"2","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130662024-01-10T18:00:31.027Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.023764745Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130672024-01-10T18:00:31.027Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.023849894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130682024-01-10T18:00:31.027Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.023931537Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130692024-01-10T18:00:31.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.024024505Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130702024-01-10T18:00:31.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.024070818Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130712024-01-10T18:00:31.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.024113581Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130722024-01-10T18:00:31.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.024293755Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"0","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130732024-01-10T18:00:31.028Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.024345612Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"0","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130742024-01-10T18:00:31.028Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.024370131Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"2","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130752024-01-10T18:00:31.028Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.024392608Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"2","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130762024-01-10T18:00:31.028Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.024651903Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"1","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130772024-01-10T18:00:31.028Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.024724962Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"1","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130782024-01-10T18:00:31.028Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.025358192Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"2","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130792024-01-10T18:00:31.028Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.025613796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"0","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130802024-01-10T18:00:31.028Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.02606878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"1","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130812024-01-10T18:00:31.029Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.026458241Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"2","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130822024-01-10T18:00:31.029Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.026522488Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"2","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130832024-01-10T18:00:31.029Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.026583866Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130842024-01-10T18:00:31.029Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.026639298Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"0","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130852024-01-10T18:00:31.029Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.026680385Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"0","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130862024-01-10T18:00:31.029Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.026726941Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130872024-01-10T18:00:31.029Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.02692198Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"1","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130882024-01-10T18:00:31.029Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027018864Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"1","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130892024-01-10T18:00:31.029Z{"msg":"[0]R flush_numbers[0..12]: [219, 215, 214, 213, 222, 222, 204, 219, 217, 215, 221, 221]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027109032Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130902024-01-10T18:00:31.029Z{"msg":"[0]R generation[0..12]: [119, 116, 114, 113, 121, 121, 107, 119, 117, 116, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027185945Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130912024-01-10T18:00:31.029Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027259916Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130922024-01-10T18:00:31.029Z{"msg":"[1]R flush_numbers[0..12]: [219, 215, 214, 213, 222, 222, 204, 219, 217, 215, 221, 221]","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO
130932024-01-10T18:00:31.031Z:30,"time":"2024-01-10T18:00:31.02733418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130942024-01-10T18:00:31.031Z{"msg":"[1]R generation[0..12]: [119, 116, 114, 113, 121, 121, 107, 119, 117, 116, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027413622Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130952024-01-10T18:00:31.031Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027464342Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130962024-01-10T18:00:31.031Z{"msg":"[2]R flush_numbers[0..12]: [219, 215, 214, 213, 222, 222, 204, 219, 217, 215, 221, 221]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027541428Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130972024-01-10T18:00:31.031Z{"msg":"[2]R generation[0..12]: [119, 116, 114, 113, 121, 121, 107, 119, 117, 116, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027627391Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130982024-01-10T18:00:31.031Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027677138Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
130992024-01-10T18:00:31.031Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027731889Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131002024-01-10T18:00:31.031Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027775401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131012024-01-10T18:00:31.031Z{"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027833535Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131022024-01-10T18:00:31.031Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027874673Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"":"downstairs","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131032024-01-10T18:00:31.031Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027929725Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131042024-01-10T18:00:31.031Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.027968093Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"0","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131052024-01-10T18:00:31.031Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131062024-01-10T18:00:31.031Z,"time":"2024-01-10T18:00:31.028028869Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"1","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131072024-01-10T18:00:31.031Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.028072188Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"client":"2","":"downstairs","session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131082024-01-10T18:00:31.031Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.028124357Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131092024-01-10T18:00:31.031Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.028162604Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131102024-01-10T18:00:31.031Z{"msg":"9a8ded25-3d02-4984-82a4-11057a7e382e is now active with session: 6420e5b2-ae66-47d5-9cdc-34ac162bd4ed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.028215851Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131112024-01-10T18:00:31.031Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.02825415Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646,"session_id":"6420e5b2-ae66-47d5-9cdc-34ac162bd4ed"}
131122024-01-10T18:00:31.032Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:31.028305155Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1646}
131132024-01-10T18:00:31.032ZLoading write count information from file "/tmp/test_repair_verify.data"
131142024-01-10T18:00:31.032ZRun Repair workload
131152024-01-10T18:00:31.039Z01/30 Write block 446 len 32768 data: 13 12 10 10 10 8 10 14
131162024-01-10T18:00:31.039Z02/30 Write block 256 len 28672 data: 7 7 10 9 8 8 6
131172024-01-10T18:00:31.039Z03/30 Write block 385 len 32768 data: 6 6 6 5 6 8 8 9
131182024-01-10T18:00:31.039Z04/30 Flush
131192024-01-10T18:00:31.039Z05/30 Write block 83 len 12288 data: 11 8 7
131202024-01-10T18:00:31.039Z06/30 Write block 65 len 24576 data: 10 12 10 9 8 9
131212024-01-10T18:00:31.039Z07/30 Write block 459 len 4096 data: 8
131222024-01-10T18:00:31.039Z08/30 Read block 385 len 32768
131232024-01-10T18:00:31.043Z09/30 Write block 539 len 28672 data: 9 11 12 9 9 10 11
131242024-01-10T18:00:31.043Z10/30 Read block 321 len 12288
131252024-01-10T18:00:31.043Z11/30 Read block 419 len 16384
131262024-01-10T18:00:31.046Z12/30 Write block 433 len 24576 data: 5 5 4 5 6 7
131272024-01-10T18:00:31.046Z13/30 Read block 439 len 36864
131282024-01-10T18:00:31.050Z14/30 Read block 297 len 16384
131292024-01-10T18:00:31.053Z15/30 Write block 479 len 24576 data: 3 4 4 4 6 9
131302024-01-10T18:00:31.053Z16/30 Write block 429 len 36864 data: 8 6 5 5 6 6 5 6 7
131312024-01-10T18:00:31.053Z17/30 Write block 311 len 20480 data: 6 3 4 5 8
131322024-01-10T18:00:31.057Z18/30 Read block 302 len 36864
131332024-01-10T18:00:31.061Z19/30 Write block 351 len 20480 data: 7 8 8 9 7
131342024-01-10T18:00:31.062Z20/30 Read block 114 len 20480
131352024-01-10T18:00:31.062Z21/30 Write block 565 len 28672 data: 9 6 8 9 9 7 7
131362024-01-10T18:00:31.066Z22/30 Read block 377 len 16384
131372024-01-10T18:00:31.066Z23/30 Write block 189 len 28672 data: 14 13 13 12 13 14 16
131382024-01-10T18:00:31.066Z24/30 Read block 73 len 20480
131392024-01-10T18:00:31.071Z25/30 Write block 505 len 8192 data: 10 12
131402024-01-10T18:00:31.074Z26/30 Write block 441 len 8192 data: 10 7
131412024-01-10T18:00:31.074Z27/30 Read block 283 len 36864
131422024-01-10T18:00:31.078Z28/30 Read block 464 len 4096
131432024-01-10T18:00:31.078Z29/30 Write block 167 len 4096 data: 13
131442024-01-10T18:00:31.078Z30/30 Write block 295 len 36864 data: 10 10 14 15 13 11 10 10 11
131452024-01-10T18:00:31.078Z----------------------------------------------------------------
131462024-01-10T18:00:31.078Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
131472024-01-10T18:00:31.079ZStates: Active Active Active
131482024-01-10T18:00:31.079ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
131492024-01-10T18:00:31.079Z 1 Acked 1000 Write 8 Done Sent Done false
131502024-01-10T18:00:31.079Z 2 Acked 1001 Write 7 Done Sent Done false
131512024-01-10T18:00:31.079Z 3 Acked 1002 Write 8 Done Sent Done false
131522024-01-10T18:00:31.079Z 4 Acked 1003 Flush 0 Done Sent Done false
131532024-01-10T18:00:31.079Z 5 Acked 1004 Write 3 Done Sent Done false
131542024-01-10T18:00:31.079Z 6 Acked 1005 Write 6 Done Sent Done false
131552024-01-10T18:00:31.079Z 7 Acked 1006 Write 1 Done Sent Done false
131562024-01-10T18:00:31.079Z 8 Acked 1007 Read 8 Done Sent Done false
131572024-01-10T18:00:31.079Z 9 Acked 1008 Write 7 Done Sent Done false
131582024-01-10T18:00:31.079Z 10 Acked 1009 Read 3 Done Sent Done false
131592024-01-10T18:00:31.079Z 11 Acked 1010 Read 4 Done Sent Done false
131602024-01-10T18:00:31.079Z 12 Acked 1011 Write 6 Done Sent Done false
131612024-01-10T18:00:31.079Z 13 Acked 1012 Read 9 Done Sent Done false
131622024-01-10T18:00:31.079Z 14 Acked 1013 Read 4 Done Sent Done false
131632024-01-10T18:00:31.079Z 15 Acked 1014 Write 6 Done Sent Done false
131642024-01-10T18:00:31.080Z 16 Acked 1015 Write 9 Done Sent Done false
131652024-01-10T18:00:31.080Z 17 Acked 1016 Write 5 Done Sent Done false
131662024-01-10T18:00:31.080Z 18 Acked 1017 Read 9 Done Sent Done false
131672024-01-10T18:00:31.080Z 19 Acked 1018 Write 5 Done Sent Done false
131682024-01-10T18:00:31.080Z 20 Acked 1019 Read 5 Done Sent Done false
131692024-01-10T18:00:31.080Z 21 Acked 1020 Write 7 Done Sent Done false
131702024-01-10T18:00:31.080Z 22 Acked 1021 Read 4 Done Sent Done false
131712024-01-10T18:00:31.080Z 23 Acked 1022 Write 7 Done Sent Done false
131722024-01-10T18:00:31.080Z 24 Acked 1023 Read 5 Done Sent Done false
131732024-01-10T18:00:31.080Z 25 Acked 1024 Write 2 Done Sent Done false
131742024-01-10T18:00:31.080Z 26 Acked 1025 Write 2 Done Sent Done false
131752024-01-10T18:00:31.081Z 27 Acked 1026 Read 9 Done Sent Done false
131762024-01-10T18:00:31.081Z 28 Acked 1027 Read 1 Done Sent Done false
131772024-01-10T18:00:31.081Z 29 Acked 1028 Write 1 Done Sent Done false
131782024-01-10T18:00:31.081Z 30 Acked 1029 Write 9 Sent Sent Sent false
131792024-01-10T18:00:31.081Z STATES DS:0 DS:1 DS:2 TOTAL
131802024-01-10T18:00:31.081Z New 0 0 0 0
131812024-01-10T18:00:31.081Z Sent 1 30 1 32
131822024-01-10T18:00:31.081Z Done 29 0 29 58
131832024-01-10T18:00:31.081Z Skipped 0 0 0 0
131842024-01-10T18:00:31.081Z Error 0 0 0 0
131852024-01-10T18:00:31.081ZLast Flush: 1003 0 1003
131862024-01-10T18:00:31.081ZDownstairs last five completed:
131872024-01-10T18:00:31.081ZUpstairs last five completed: 30 29 28 27 26
131882024-01-10T18:00:31.082ZWrote out file "/tmp/test_repair_verify.data"
131892024-01-10T18:00:31.085Z
131902024-01-10T18:00:31.091ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
131912024-01-10T18:00:31.124ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131922024-01-10T18:00:31.124Z 0 000-019 119 119 119 219 219 219 F F F
131932024-01-10T18:00:31.124Z 1 020-039 116 116 116 215 215 215 F F F
131942024-01-10T18:00:31.124Z 2 040-059 114 114 114 214 214 214 F F F
131952024-01-10T18:00:31.124Z 3 060-079 113 113 113 213 213 213 T F T <---
131962024-01-10T18:00:31.124Z 4 080-099 121 121 121 222 222 222 T F T <---
131972024-01-10T18:00:31.124Z 5 100-119 121 121 121 222 222 222 F F F
131982024-01-10T18:00:31.124Z 6 120-139 107 107 107 204 204 204 F F F
131992024-01-10T18:00:31.128Z 7 140-159 119 119 119 219 219 219 F F F
132002024-01-10T18:00:31.128Z 8 160-179 117 117 117 217 217 217 T F T <---
132012024-01-10T18:00:31.128Z 9 180-199 116 116 116 215 215 215 T F T <---
132022024-01-10T18:00:31.128Z 10 200-219 120 120 120 221 221 221 F F F
132032024-01-10T18:00:31.128Z 11 220-239 120 120 120 221 221 221 F F F
132042024-01-10T18:00:31.128Z 12 240-259 122 121 122 223 222 223 F F F <---
132052024-01-10T18:00:31.128Z 13 260-279 122 117 122 223 217 223 F F F <---
132062024-01-10T18:00:31.128Z 14 280-299 121 121 121 222 222 222 T F T <---
132072024-01-10T18:00:31.128Z 15 300-319 116 116 116 215 215 215 T F T <---
132082024-01-10T18:00:31.129Z 16 320-339 119 119 119 219 219 219 F F F
132092024-01-10T18:00:31.129Z 17 340-359 117 117 117 217 217 217 T F T <---
132102024-01-10T18:00:31.129Z 18 360-379 121 121 121 222 222 222 F F F
132112024-01-10T18:00:31.129Z 19 380-399 122 121 122 223 222 223 F F F <---
132122024-01-10T18:00:31.130Z 20 400-419 121 121 121 222 222 222 F F F
132132024-01-10T18:00:31.130Z 21 420-439 121 121 121 222 222 222 T F T <---
132142024-01-10T18:00:31.130Z 22 440-459 122 119 122 223 219 223 T F T <---
132152024-01-10T18:00:31.130Z 23 460-479 119 119 119 219 219 219 T F T <---
132162024-01-10T18:00:31.130Z 24 480-499 121 121 121 222 222 222 T F T <---
132172024-01-10T18:00:31.130Z 25 500-519 119 119 119 219 219 219 T F T <---
132182024-01-10T18:00:31.130Z 26 520-539 119 119 119 219 219 219 T F T <---
132192024-01-10T18:00:31.130Z 27 540-559 121 121 121 222 222 222 T F T <---
132202024-01-10T18:00:31.130Z 28 560-579 116 116 116 216 216 216 T F T <---
132212024-01-10T18:00:31.130Z 29 580-599 116 116 116 216 216 216 F F F
132222024-01-10T18:00:31.130ZMax gen: 122, Max flush: 223
132232024-01-10T18:00:31.130ZError: Difference in extent metadata found!
132242024-01-10T18:00:31.130ZA Difference in extent metadata is expected here
132252024-01-10T18:00:31.130ZOn loop 61
132262024-01-10T18:00:31.130Z
132272024-01-10T18:00:31.130ZVerifying data now
132282024-01-10T18:00:36.795ZNo change after verify
132292024-01-10T18:00:36.796ZLoop: 61 Downstairs dump after verify (and repair):
132302024-01-10T18:00:36.817ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132312024-01-10T18:00:36.817Z 0 000-019 119 119 119 219 219 219 F F F
132322024-01-10T18:00:36.818Z 1 020-039 116 116 116 215 215 215 F F F
132332024-01-10T18:00:36.818Z 2 040-059 114 114 114 214 214 214 F F F
132342024-01-10T18:00:36.818Z 3 060-079 123 123 123 224 224 224 F F F
132352024-01-10T18:00:36.818Z 4 080-099 123 123 123 224 224 224 F F F
132362024-01-10T18:00:36.821Z 5 100-119 121 121 121 222 222 222 F F F
132372024-01-10T18:00:36.821Z 6 120-139 107 107 107 204 204 204 F F F
132382024-01-10T18:00:36.821Z 7 140-159 119 119 119 219 219 219 F F F
132392024-01-10T18:00:36.822Z 8 160-179 123 123 123 224 224 224 F F F
132402024-01-10T18:00:36.822Z 9 180-199 123 123 123 224 224 224 F F F
132412024-01-10T18:00:36.823Z 10 200-219 120 120 120 221 221 221 F F F
132422024-01-10T18:00:36.823Z 11 220-239 120 120 120 221 221 221 F F F
132432024-01-10T18:00:36.823Z 12 240-259 122 122 122 223 223 223 F F F
132442024-01-10T18:00:36.824Z 13 260-279 122 122 122 223 223 223 F F F
132452024-01-10T18:00:36.824Z 14 280-299 123 123 123 224 224 224 F F F
132462024-01-10T18:00:36.824Z 15 300-319 123 123 123 224 224 224 F F F
132472024-01-10T18:00:36.824Z 16 320-339 119 119 119 219 219 219 F F F
132482024-01-10T18:00:36.824Z 17 340-359 123 123 123 224 224 224 F F F
132492024-01-10T18:00:36.824Z 18 360-379 121 121 121 222 222 222 F F F
132502024-01-10T18:00:36.824Z 19 380-399 122 122 122 223 223 223 F F F
132512024-01-10T18:00:36.824Z 20 400-419 121 121 121 222 222 222 F F F
132522024-01-10T18:00:36.824Z 21 420-439 123 123 123 224 224 224 F F F
132532024-01-10T18:00:36.824Z 22 440-459 123 123 123 224 224 224 F F F
132542024-01-10T18:00:36.824Z 23 460-479 123 123 123 224 224 224 F F F
132552024-01-10T18:00:36.824Z 24 480-499 123 123 123 224 224 224 F F F
132562024-01-10T18:00:36.824Z 25 500-519 123 123 123 224 224 224 F F F
132572024-01-10T18:00:36.824Z 26 520-539 123 123 123 224 224 224 F F F
132582024-01-10T18:00:36.824Z 27 540-559 123 123 123 224 224 224 F F F
132592024-01-10T18:00:36.824Z 28 560-579 123 123 123 224 224 224 F F F
132602024-01-10T18:00:36.824Z 29 580-599 116 116 116 216 216 216 F F F
132612024-01-10T18:00:36.824ZMax gen: 123, Max flush: 224
132622024-01-10T18:00:36.827Z
132632024-01-10T18:00:36.827ZBegin loop 62
132642024-01-10T18:00:36.827ZDownstairs to restart: 1
132652024-01-10T18:00:36.852Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.85732674Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654}
132662024-01-10T18:00:36.852Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.857812592Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654}
132672024-01-10T18:00:36.928Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.857890698Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654}
132682024-01-10T18:00:36.928Z{"msg":"Crucible 96b064bc-57ca-4af4-bda6-e406431177ec has session id: 775347e6-80f5-4138-b6b8-16a9fb872af3","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.857935953Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132692024-01-10T18:00:36.928Z{"msg":"Upstairs opts: Upstairs UUID: 96b064bc-57ca-4af4-bda6-e406431177ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.857993147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132702024-01-10T18:00:36.928Z{"msg":"Crucible stats registered with UUID: 96b064bc-57ca-4af4-bda6-e406431177ec","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.858064351Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132712024-01-10T18:00:36.928ZCrucible runtime is spawned
132722024-01-10T18:00:36.928Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.858328276Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654}
132732024-01-10T18:00:36.928Z{"msg":"96b064bc-57ca-4af4-bda6-e406431177ec active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.858527306Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132742024-01-10T18:00:36.928Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.858609714Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"0","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132752024-01-10T18:00:36.928Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.858655651Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"0","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132762024-01-10T18:00:36.930Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.858745342Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"1","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132772024-01-10T18:00:36.930Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.858881971Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"1","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132782024-01-10T18:00:36.930Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.858959418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"2","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132792024-01-10T18:00:36.930Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.859116344Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"2","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132802024-01-10T18:00:36.931Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.859228473Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132812024-01-10T18:00:36.931Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.859306617Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132822024-01-10T18:00:36.931Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.8593491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132832024-01-10T18:00:36.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.859375344Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132842024-01-10T18:00:36.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.85939783Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132852024-01-10T18:00:36.932Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.859441827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132862024-01-10T18:00:36.933Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.860595684Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"0","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132872024-01-10T18:00:36.933Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.860662772Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"0","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132882024-01-10T18:00:36.933Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.860688021Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"2","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132892024-01-10T18:00:36.933Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.860710196Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"2","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132902024-01-10T18:00:36.940Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.861783828Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"2","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132912024-01-10T18:00:36.940Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.861886395Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"0","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132922024-01-10T18:00:36.940Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.862274391Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"1","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132932024-01-10T18:00:36.941Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.862326802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"1","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132942024-01-10T18:00:36.941Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.862814433Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"2","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132952024-01-10T18:00:36.941Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.862894023Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"2","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132962024-01-10T18:00:36.942Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.862966283Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132972024-01-10T18:00:36.942Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.863005265Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"0","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132982024-01-10T18:00:36.942Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.863072528Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"0","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
132992024-01-10T18:00:36.942Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.863100379Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133002024-01-10T18:00:36.942Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.863356914Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"1","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133012024-01-10T18:00:36.942Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864015328Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"1","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133022024-01-10T18:00:36.942Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.86409306Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"1","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133032024-01-10T18:00:36.942Z{"msg":"[0]R flush_numbers[0..12]: [219, 215, 214, 224, 224, 222, 204, 219, 224, 224, 221, 221]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864157857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133042024-01-10T18:00:36.942Z{"msg":"[0]R generation[0..12]: [119, 116, 114, 123, 123, 121, 107, 119, 123, 123, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864186412Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133052024-01-10T18:00:36.942Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864229237Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133062024-01-10T18:00:36.948Z{"msg":"[1]R flush_numbers[0..12]: [219, 215, 214, 224, 224, 222, 204, 219, 224, 224, 221, 221]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864253796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133072024-01-10T18:00:36.948Z{"msg":"[1]R generation[0..12]: [119, 116, 114, 123, 123, 121, 107, 119, 123, 123, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864277959Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133082024-01-10T18:00:36.948Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864301458Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pidWait for a query_work_queue command to finish before sending IO
133092024-01-10T18:00:36.948Z":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133102024-01-10T18:00:36.948Z{"msg":"[2]R flush_numbers[0..12]: [219, 215, 214, 224, 224, 222, 204, 219, 224, 224, 221, 221]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864420581Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133112024-01-10T18:00:36.948Z{"msg":"[2]R generation[0..12]: [119, 116, 114, 123, 123, 121, 107, 119, 123, 123, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864474282Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133122024-01-10T18:00:36.948Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864499878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133132024-01-10T18:00:36.948Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864541829Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133142024-01-10T18:00:36.948Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864565017Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133152024-01-10T18:00:36.948Z{"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864604181Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133162024-01-10T18:00:36.949Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864627028Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"":"downstairs","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133172024-01-10T18:00:36.949Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864666867Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133182024-01-10T18:00:36.949Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864687994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"0","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133192024-01-10T18:00:36.949Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864735024Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"1","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133202024-01-10T18:00:36.949Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864757895Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"client":"2","":"downstairs","session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133212024-01-10T18:00:36.949Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864799127Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133222024-01-10T18:00:36.949Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864820112Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133232024-01-10T18:00:36.949Z{"msg":"96b064bc-57ca-4af4-bda6-e406431177ec is now active with session: 775347e6-80f5-4138-b6b8-16a9fb872af3","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.86484133Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133242024-01-10T18:00:36.949Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864878666Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654,"session_id":"775347e6-80f5-4138-b6b8-16a9fb872af3"}
133252024-01-10T18:00:36.949Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:36.864899698Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1654}
133262024-01-10T18:00:36.949ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133272024-01-10T18:00:36.949ZLoading write count information from file "/tmp/test_repair_verify.data"
133282024-01-10T18:00:36.949ZRun Repair workload
133292024-01-10T18:00:36.949Z01/30 Write block 354 len 4096 data: 10
133302024-01-10T18:00:36.949Z02/30 Read block 308 len 40960
133312024-01-10T18:00:36.949Z03/30 Write block 298 len 20480 data: 16 14 12 11 11
133322024-01-10T18:00:36.950Z04/30 Write block 194 len 36864 data: 15 17 15 15 16 16 15 14 13
133332024-01-10T18:00:36.950Z05/30 Write block 89 len 12288 data: 7 8 11
133342024-01-10T18:00:36.950Z06/30 Read block 10 len 24576
133352024-01-10T18:00:36.950Z07/30 Read block 260 len 4096
133362024-01-10T18:00:36.950Z08/30 Read block 122 len 12288
133372024-01-10T18:00:36.950Z09/30 Flush
133382024-01-10T18:00:36.950Z10/30 Write block 326 len 12288 data: 8 8 8
133392024-01-10T18:00:36.950Z11/30 Write block 119 len 16384 data: 9 9 8 8
133402024-01-10T18:00:36.950Z12/30 Write block 407 len 4096 data: 15
133412024-01-10T18:00:36.950Z13/30 Read block 275 len 12288
133422024-01-10T18:00:36.950Z14/30 Write block 85 len 12288 data: 8 7 7
133432024-01-10T18:00:36.950Z15/30 Read block 456 len 32768
133442024-01-10T18:00:36.950Z16/30 Read block 408 len 36864
133452024-01-10T18:00:36.950Z17/30 Flush
133462024-01-10T18:00:36.950Z18/30 Write block 570 len 8192 data: 8 8
133472024-01-10T18:00:36.950Z19/30 Read block 558 len 36864
133482024-01-10T18:00:36.950Z20/30 Read block 223 len 24576
133492024-01-10T18:00:36.950Z21/30 Write block 52 len 36864 data: 10 8 11 10 10 9 11 11 10
133502024-01-10T18:00:36.950Z22/30 Read block 301 len 28672
133512024-01-10T18:00:36.950Z23/30 Read block 265 len 40960
133522024-01-10T18:00:36.950Z24/30 Read block 180 len 24576
133532024-01-10T18:00:36.950Z25/30 Read block 145 len 40960
133542024-01-10T18:00:36.950Z26/30 Read block 368 len 36864
133552024-01-10T18:00:36.951Z27/30 Write block 283 len 40960 data: 9 10 12 10 9 8 11 13 14 15
133562024-01-10T18:00:36.951Z28/30 Read block 306 len 32768
133572024-01-10T18:00:36.951Z29/30 Write block 475 len 40960 data: 5 5 5 2 4 5 5 5 7 10
133582024-01-10T18:00:36.951Z30/30 Write block 36 len 8192 data: 12 10
133592024-01-10T18:00:36.951Z----------------------------------------------------------------
133602024-01-10T18:00:36.951Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
133612024-01-10T18:00:36.951ZStates: Active Active Active
133622024-01-10T18:00:36.951ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
133632024-01-10T18:00:36.951Z 1 Acked 1000 Write 1 Done Done Done false
133642024-01-10T18:00:36.951Z 2 Acked 1001 Read 10 Done Done Done false
133652024-01-10T18:00:36.951Z 3 Acked 1002 Write 5 Done Sent Done false
133662024-01-10T18:00:36.951Z 4 Acked 1003 Write 9 Done Sent Done false
133672024-01-10T18:00:36.951Z 5 Acked 1004 Write 3 Done Sent Done false
133682024-01-10T18:00:36.951Z 6 Acked 1005 Read 6 Done Sent Done false
133692024-01-10T18:00:36.951Z 7 Acked 1006 Read 1 Done Sent Done false
133702024-01-10T18:00:36.951Z 8 Acked 1007 Read 3 Done Sent Done false
133712024-01-10T18:00:36.951Z 9 Acked 1008 Flush 0 Done Sent Done false
133722024-01-10T18:00:36.951Z 10 Acked 1009 Write 3 Done Sent Done false
133732024-01-10T18:00:36.951Z 11 Acked 1010 Write 4 Done Sent Done false
133742024-01-10T18:00:36.951Z 12 Acked 1011 Write 1 Done Sent Done false
133752024-01-10T18:00:36.951Z 13 Acked 1012 Read 3 Done Sent Done false
133762024-01-10T18:00:36.951Z 14 Acked 1013 Write 3 Done Sent Done false
133772024-01-10T18:00:36.952Z 15 Acked 1014 Read 8 Done Sent Done false
133782024-01-10T18:00:36.952Z 16 Acked 1015 Read 9 Done Sent Done false
133792024-01-10T18:00:36.952Z 17 Acked 1016 Flush 0 Done Sent Done false
133802024-01-10T18:00:36.952Z 18 Acked 1017 Write 2 Done Sent Done false
133812024-01-10T18:00:36.952Z 19 Acked 1018 Read 9 Done Sent Done false
133822024-01-10T18:00:36.952Z 20 Acked 1019 Read 6 Done Sent Done false
133832024-01-10T18:00:36.952Z 21 Acked 1020 Write 9 Done Sent Done false
133842024-01-10T18:00:36.952Z 22 Acked 1021 Read 7 Done Sent Done false
133852024-01-10T18:00:36.952Z 23 Acked 1022 Read 10 Done Sent Done false
133862024-01-10T18:00:36.952Z 24 Acked 1023 Read 6 Done Sent Done false
133872024-01-10T18:00:36.952Z 25 Acked 1024 Read 10 Done Sent Done false
133882024-01-10T18:00:36.952Z 26 Acked 1025 Read 9 Done Sent Done false
133892024-01-10T18:00:36.953Z 27 Acked 1026 Write 10 Done Sent Done false
133902024-01-10T18:00:36.957Z 28 Acked 1027 Read 8 Done Sent Done false
133912024-01-10T18:00:36.957Z 29 Acked 1028 Write 10 Sent Sent Sent false
133922024-01-10T18:00:36.957Z 30 Acked 1029 Write 2 Sent Sent Sent false
133932024-01-10T18:00:36.957Z STATES DS:0 DS:1 DS:2 TOTAL
133942024-01-10T18:00:36.957Z New 0 0 0 0
133952024-01-10T18:00:36.957Z Sent 2 28 2 32
133962024-01-10T18:00:36.957Z Done 28 2 28 58
133972024-01-10T18:00:36.957Z Skipped 0 0 0 0
133982024-01-10T18:00:36.957Z Error 0 0 0 0
133992024-01-10T18:00:36.957ZLast Flush: 1016 0 1016
134002024-01-10T18:00:36.957ZDownstairs last five completed:
134012024-01-10T18:00:36.957ZUpstairs last five completed: 30 29 28 27 26
134022024-01-10T18:00:36.957ZWrote out file "/tmp/test_repair_verify.data"
134032024-01-10T18:00:36.958Z
134042024-01-10T18:00:36.958ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134052024-01-10T18:00:36.965ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134062024-01-10T18:00:36.965Z 0 000-019 119 119 119 219 219 219 F F F
134072024-01-10T18:00:36.965Z 1 020-039 116 116 116 215 215 215 T F T <---
134082024-01-10T18:00:36.965Z 2 040-059 114 114 114 214 214 214 T F T <---
134092024-01-10T18:00:36.965Z 3 060-079 123 123 123 224 224 224 T F T <---
134102024-01-10T18:00:36.965Z 4 080-099 124 123 124 226 224 226 F F F <---
134112024-01-10T18:00:36.965Z 5 100-119 124 121 124 226 222 226 F F F <---
134122024-01-10T18:00:36.965Z 6 120-139 124 107 124 226 204 226 F F F <---
134132024-01-10T18:00:36.965Z 7 140-159 119 119 119 219 219 219 F F F
134142024-01-10T18:00:36.965Z 8 160-179 123 123 123 224 224 224 F F F
134152024-01-10T18:00:36.966Z 9 180-199 124 123 124 225 224 225 F F F <---
134162024-01-10T18:00:36.966Z 10 200-219 124 120 124 225 221 225 F F F <---
134172024-01-10T18:00:36.966Z 11 220-239 120 120 120 221 221 221 F F F
134182024-01-10T18:00:36.966Z 12 240-259 122 122 122 223 223 223 F F F
134192024-01-10T18:00:36.966Z 13 260-279 122 122 122 223 223 223 F F F
134202024-01-10T18:00:36.966Z 14 280-299 124 123 124 225 224 225 T F T <---
134212024-01-10T18:00:36.966Z 15 300-319 124 123 124 225 224 225 F F F <---
134222024-01-10T18:00:36.966Z 16 320-339 124 119 124 226 219 226 F F F <---
134232024-01-10T18:00:36.966Z 17 340-359 124 123 124 225 224 225 F T F <---
134242024-01-10T18:00:36.966Z 18 360-379 121 121 121 222 222 222 F F F
134252024-01-10T18:00:36.966Z 19 380-399 122 122 122 223 223 223 F F F
134262024-01-10T18:00:36.967Z 20 400-419 124 121 124 226 222 226 F F F <---
134272024-01-10T18:00:36.967Z 21 420-439 123 123 123 224 224 224 F F F
134282024-01-10T18:00:36.967Z 22 440-459 123 123 123 224 224 224 F F F
134292024-01-10T18:00:36.967Z 23 460-479 123 123 123 224 224 224 T F T <---
134302024-01-10T18:00:36.967Z 24 480-499 123 123 123 224 224 224 T F T <---
134312024-01-10T18:00:36.967Z 25 500-519 123 123 123 224 224 224 F F F
134322024-01-10T18:00:36.967Z 26 520-539 123 123 123 224 224 224 F F F
134332024-01-10T18:00:36.967Z 27 540-559 123 123 123 224 224 224 F F F
134342024-01-10T18:00:36.967Z 28 560-579 123 123 123 224 224 224 T F T <---
134352024-01-10T18:00:36.967Z 29 580-599 116 116 116 216 216 216 F F F
134362024-01-10T18:00:36.967ZMax gen: 124, Max flush: 226
134372024-01-10T18:00:36.967ZError: Difference in extent metadata found!
134382024-01-10T18:00:36.970ZA Difference in extent metadata is expected here
134392024-01-10T18:00:36.970ZOn loop 62
134402024-01-10T18:00:36.970Z
134412024-01-10T18:00:36.974ZVerifying data now
134422024-01-10T18:00:42.454ZNo change after verify
134432024-01-10T18:00:42.454ZLoop: 62 Downstairs dump after verify (and repair):
134442024-01-10T18:00:42.477ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134452024-01-10T18:00:42.477Z 0 000-019 119 119 119 219 219 219 F F F
134462024-01-10T18:00:42.477Z 1 020-039 125 125 125 227 227 227 F F F
134472024-01-10T18:00:42.477Z 2 040-059 125 125 125 227 227 227 F F F
134482024-01-10T18:00:42.477Z 3 060-079 125 125 125 227 227 227 F F F
134492024-01-10T18:00:42.482Z 4 080-099 124 124 124 226 226 226 F F F
134502024-01-10T18:00:42.482Z 5 100-119 124 124 124 226 226 226 F F F
134512024-01-10T18:00:42.482Z 6 120-139 124 124 124 226 226 226 F F F
134522024-01-10T18:00:42.482Z 7 140-159 119 119 119 219 219 219 F F F
134532024-01-10T18:00:42.482Z 8 160-179 123 123 123 224 224 224 F F F
134542024-01-10T18:00:42.482Z 9 180-199 124 124 124 225 225 225 F F F
134552024-01-10T18:00:42.482Z 10 200-219 124 124 124 225 225 225 F F F
134562024-01-10T18:00:42.482Z 11 220-239 120 120 120 221 221 221 F F F
134572024-01-10T18:00:42.482Z 12 240-259 122 122 122 223 223 223 F F F
134582024-01-10T18:00:42.482Z 13 260-279 122 122 122 223 223 223 F F F
134592024-01-10T18:00:42.482Z 14 280-299 125 125 125 227 227 227 F F F
134602024-01-10T18:00:42.482Z 15 300-319 124 124 124 225 225 225 F F F
134612024-01-10T18:00:42.482Z 16 320-339 124 124 124 226 226 226 F F F
134622024-01-10T18:00:42.482Z 17 340-359 124 124 124 225 225 225 F F F
134632024-01-10T18:00:42.482Z 18 360-379 121 121 121 222 222 222 F F F
134642024-01-10T18:00:42.482Z 19 380-399 122 122 122 223 223 223 F F F
134652024-01-10T18:00:42.482Z 20 400-419 124 124 124 226 226 226 F F F
134662024-01-10T18:00:42.482Z 21 420-439 123 123 123 224 224 224 F F F
134672024-01-10T18:00:42.482Z 22 440-459 123 123 123 224 224 224 F F F
134682024-01-10T18:00:42.482Z 23 460-479 125 125 125 227 227 227 F F F
134692024-01-10T18:00:42.482Z 24 480-499 125 125 125 227 227 227 F F F
134702024-01-10T18:00:42.482Z 25 500-519 123 123 123 224 224 224 F F F
134712024-01-10T18:00:42.483Z 26 520-539 123 123 123 224 224 224 F F F
134722024-01-10T18:00:42.483Z 27 540-559 123 123 123 224 224 224 F F F
134732024-01-10T18:00:42.483Z 28 560-579 125 125 125 227 227 227 F F F
134742024-01-10T18:00:42.483Z 29 580-599 116 116 116 216 216 216 F F F
134752024-01-10T18:00:42.483ZMax gen: 125, Max flush: 227
134762024-01-10T18:00:42.483Z
134772024-01-10T18:00:42.483ZBegin loop 63
134782024-01-10T18:00:42.483ZDownstairs to restart: 0
134792024-01-10T18:00:42.523Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.516211889Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662}
134802024-01-10T18:00:42.524Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.516702856Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662}
134812024-01-10T18:00:42.529Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.516807348Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662}
134822024-01-10T18:00:42.529Z{"msg":"Crucible runtime is spawned
134832024-01-10T18:00:42.529ZCrucible f4355193-46ef-47b6-b8c2-d10aec34961b has session id: 1c7d9374-55a3-45ce-8fc4-3a8776082d29","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.51695129Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134842024-01-10T18:00:42.529Z{"msg":"Upstairs opts: Upstairs UUID: f4355193-46ef-47b6-b8c2-d10aec34961b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517074564Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134852024-01-10T18:00:42.529Z{"msg":"Crucible stats registered with UUID: f4355193-46ef-47b6-b8c2-d10aec34961b","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517122376Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134862024-01-10T18:00:42.529Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517181627Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662}
134872024-01-10T18:00:42.530Z{"msg":"f4355193-46ef-47b6-b8c2-d10aec34961b active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517219586Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134882024-01-10T18:00:42.530Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517274211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"0","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134892024-01-10T18:00:42.530Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517368809Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"0","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134902024-01-10T18:00:42.530Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517414489Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"1","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134912024-01-10T18:00:42.530Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.51745577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"1","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134922024-01-10T18:00:42.530Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517524514Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"2","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134932024-01-10T18:00:42.530Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517570854Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"2","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134942024-01-10T18:00:42.530Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517679435Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134952024-01-10T18:00:42.530Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517869006Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134962024-01-10T18:00:42.530Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.517992261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134972024-01-10T18:00:42.530Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.518041264Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134982024-01-10T18:00:42.530Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.518084688Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
134992024-01-10T18:00:42.530Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.518167298Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135002024-01-10T18:00:42.530Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.518982896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"1","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135012024-01-10T18:00:42.531Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.519057194Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"1","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135022024-01-10T18:00:42.531Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.519085848Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"2","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135032024-01-10T18:00:42.531Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.519107965Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"2","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135042024-01-10T18:00:42.531Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.519622221Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"0","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135052024-01-10T18:00:42.531Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.519683444Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"0","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135062024-01-10T18:00:42.531Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.520010919Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"1","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135072024-01-10T18:00:42.531Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.520406619Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"2","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135082024-01-10T18:00:42.531Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.521062231Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"1","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135092024-01-10T18:00:42.531Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.52110411Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"1","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135102024-01-10T18:00:42.531Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.521128216Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135112024-01-10T18:00:42.531Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.521287891Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"0","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135122024-01-10T18:00:42.531Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.521404108Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"2","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135132024-01-10T18:00:42.531Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.521481616Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"2","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135142024-01-10T18:00:42.531Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.521508931Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135152024-01-10T18:00:42.532Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522134042Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"0","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135162024-01-10T18:00:42.532Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.52220878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"0","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135172024-01-10T18:00:42.532Z{"msg":"[0]R flush_numbers[0..12]: [219, 227, 227, 227, 226, 226, 226, 219, 224, 225, 225, 221]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522273231Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135182024-01-10T18:00:42.532Z{"msg":"[0]R generation[0..12]: [119, 125, 125, 125, 124, 124, 124, 119, 123, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522301344Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135192024-01-10T18:00:42.532Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522326955Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135202024-01-10T18:00:42.532Z{"msg":"[1]R flush_numbers[0..12]: [219, 227, 227, 227, 226, 226, 226, 219, 224, 225, 225, 221]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522350906Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135212024-01-10T18:00:42.532Z{"msg":"[1]R generation[0..12]: [119, 125, 125, 125, 124, 124, 124, 119, 123, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522399512Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135222024-01-10T18:00:42.532Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522441582Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135232024-01-10T18:00:42.532Z{"msg":"[2]R flush_numbers[0..12]: [219, 227, 227, 227, 226, 226, 226, 219, 224, 225, 225, 221]","v":0,"Wait for a query_work_queue command to finish before sending IO
135242024-01-10T18:00:42.532Zname":"crucible","level":30,"time":"2024-01-10T18:00:42.522533078Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135252024-01-10T18:00:42.533Z{"msg":"[2]R generation[0..12]: [119, 125, 125, 125, 124, 124, 124, 119, 123, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522595058Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135262024-01-10T18:00:42.533Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522625512Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135272024-01-10T18:00:42.533Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522679925Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135282024-01-10T18:00:42.533Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522705286Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135292024-01-10T18:00:42.533Z{"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.52276064Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135302024-01-10T18:00:42.533Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522790901Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"":"downstairs","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135312024-01-10T18:00:42.533Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522844727Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135322024-01-10T18:00:42.533Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522867954Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"0","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135332024-01-10T18:00:42.533Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522922596Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"1","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135342024-01-10T18:00:42.534Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.522948616Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"client":"2","":"downstairs","session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135352024-01-10T18:00:42.536Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.523010543Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135362024-01-10T18:00:42.536Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.523038347Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135372024-01-10T18:00:42.536Z{"msg":"f4355193-46ef-47b6-b8c2-d10aec34961b is now active with session: 1c7d9374-55a3-45ce-8fc4-3a8776082d29","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.523091039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135382024-01-10T18:00:42.536Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.523113653Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662,"session_id":"1c7d9374-55a3-45ce-8fc4-3a8776082d29"}
135392024-01-10T18:00:42.536Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:42.523137365Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1662}
135402024-01-10T18:00:42.536ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135412024-01-10T18:00:42.536ZLoading write count information from file "/tmp/test_repair_verify.data"
135422024-01-10T18:00:42.536ZRun Repair workload
135432024-01-10T18:00:42.536Z01/30 Write block 292 len 28672 data: 16 12 10 11 11 15 17
135442024-01-10T18:00:42.536Z02/30 Read block 187 len 12288
135452024-01-10T18:00:42.536Z03/30 Read block 326 len 28672
135462024-01-10T18:00:42.536Z04/30 Write block 562 len 12288 data: 11 11 10
135472024-01-10T18:00:42.541Z05/30 Read block 447 len 40960
135482024-01-10T18:00:42.541Z06/30 Read block 160 len 36864
135492024-01-10T18:00:42.541Z07/30 Read block 227 len 32768
135502024-01-10T18:00:42.541Z08/30 Read block 72 len 24576
135512024-01-10T18:00:42.541Z09/30 Write block 232 len 28672 data: 4 4 5 5 6 8 8
135522024-01-10T18:00:42.541Z10/30 Read block 551 len 12288
135532024-01-10T18:00:42.547Z11/30 Read block 535 len 32768
135542024-01-10T18:00:42.552Z12/30 Read block 27 len 20480
135552024-01-10T18:00:42.552Z13/30 Write block 274 len 20480 data: 12 14 11 10 10
135562024-01-10T18:00:42.552Z14/30 Flush
135572024-01-10T18:00:42.552Z15/30 Flush
135582024-01-10T18:00:42.556Z16/30 Write block 315 len 16384 data: 9 8 6 7
135592024-01-10T18:00:42.556Z17/30 Read block 371 len 36864
135602024-01-10T18:00:42.561Z18/30 Write block 41 len 16384 data: 8 9 8 8
135612024-01-10T18:00:42.561Z19/30 Read block 29 len 24576
135622024-01-10T18:00:42.561Z20/30 Write block 402 len 28672 data: 9 12 13 15 16 16 13
135632024-01-10T18:00:42.566Z21/30 Flush
135642024-01-10T18:00:42.571Z22/30 Write block 323 len 28672 data: 9 11 11 9 9 9 7
135652024-01-10T18:00:42.571Z23/30 Read block 73 len 24576
135662024-01-10T18:00:42.571Z24/30 Write block 257 len 40960 data: 8 11 10 9 9 7 7 8 6 10
135672024-01-10T18:00:42.571Z25/30 Read block 127 len 8192
135682024-01-10T18:00:42.575Z26/30 Write block 531 len 40960 data: 17 12 13 13 12 12 10 10 10 12
135692024-01-10T18:00:42.575Z27/30 Write block 242 len 16384 data: 9 10 15 14
135702024-01-10T18:00:42.575Z28/30 Read block 413 len 12288
135712024-01-10T18:00:42.578Z29/30 Read block 285 len 20480
135722024-01-10T18:00:42.578Z30/30 Read block 183 len 20480
135732024-01-10T18:00:42.582Z----------------------------------------------------------------
135742024-01-10T18:00:42.582Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
135752024-01-10T18:00:42.582ZStates: Active Active Active
135762024-01-10T18:00:42.582ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
135772024-01-10T18:00:42.582Z 1 Acked 1000 Write 7 Done Done Done false
135782024-01-10T18:00:42.582Z 2 Acked 1001 Read 3 Done Done Done false
135792024-01-10T18:00:42.582Z 3 Acked 1002 Read 7 Done Done Done false
135802024-01-10T18:00:42.582Z 4 Acked 1003 Write 3 Sent Done Done false
135812024-01-10T18:00:42.582Z 5 Acked 1004 Read 10 Sent Done Done false
135822024-01-10T18:00:42.583Z 6 Acked 1005 Read 9 Sent Done Done false
135832024-01-10T18:00:42.583Z 7 Acked 1006 Read 8 Sent Done Done false
135842024-01-10T18:00:42.583Z 8 Acked 1007 Read 6 Sent Done Done false
135852024-01-10T18:00:42.583Z 9 Acked 1008 Write 7 Sent Done Done false
135862024-01-10T18:00:42.583Z 10 Acked 1009 Read 3 Sent Done Done false
135872024-01-10T18:00:42.583Z 11 Acked 1010 Read 8 Sent Done Done false
135882024-01-10T18:00:42.583Z 12 Acked 1011 Read 5 Sent Done Done false
135892024-01-10T18:00:42.583Z 13 Acked 1012 Write 5 Sent Done Done false
135902024-01-10T18:00:42.583Z 14 Acked 1013 Flush 0 Sent Done Done false
135912024-01-10T18:00:42.583Z 15 Acked 1014 Flush 0 Sent Done Done false
135922024-01-10T18:00:42.583Z 16 Acked 1015 Write 4 Sent Done Done false
135932024-01-10T18:00:42.583Z 17 Acked 1016 Read 9 Sent Done Done false
135942024-01-10T18:00:42.583Z 18 Acked 1017 Write 4 Sent Done Done false
135952024-01-10T18:00:42.583Z 19 Acked 1018 Read 6 Sent Done Done false
135962024-01-10T18:00:42.583Z 20 Acked 1019 Write 7 Sent Done Done false
135972024-01-10T18:00:42.583Z 21 Acked 1020 Flush 0 Sent Done Done false
135982024-01-10T18:00:42.583Z 22 Acked 1021 Write 7 Sent Done Done false
135992024-01-10T18:00:42.583Z 23 Acked 1022 Read 6 Sent Done Done false
136002024-01-10T18:00:42.583Z 24 Acked 1023 Write 10 Sent Done Done false
136012024-01-10T18:00:42.584Z 25 Acked 1024 Read 2 Sent Done Done false
136022024-01-10T18:00:42.584Z 26 Acked 1025 Write 10 Sent Done Done false
136032024-01-10T18:00:42.584Z 27 Acked 1026 Write 4 Sent Done Done false
136042024-01-10T18:00:42.584Z 28 Acked 1027 Read 3 Sent Done Done false
136052024-01-10T18:00:42.584Z 29 Acked 1028 Read 5 Sent Done Done false
136062024-01-10T18:00:42.584Z 30 Acked 1029 Read 5 Sent Done Done false
136072024-01-10T18:00:42.584Z STATES DS:0 DS:1 DS:2 TOTAL
136082024-01-10T18:00:42.584Z New 0 0 0 0
136092024-01-10T18:00:42.584Z Sent 27 0 0 27
136102024-01-10T18:00:42.584Z Done 3 30 30 63
136112024-01-10T18:00:42.584Z Skipped 0 0 0 0
136122024-01-10T18:00:42.584Z Error 0 0 0 0
136132024-01-10T18:00:42.584ZLast Flush: 0 1020 1020
136142024-01-10T18:00:42.585ZDownstairs last five completed:
136152024-01-10T18:00:42.585ZUpstairs last five completed: 30 29 28 27 26
136162024-01-10T18:00:42.585ZWrote out file "/tmp/test_repair_verify.data"
136172024-01-10T18:00:42.588Z
136182024-01-10T18:00:42.592ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136192024-01-10T18:00:42.616ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136202024-01-10T18:00:42.616Z 0 000-019 119 119 119 219 219 219 F F F
136212024-01-10T18:00:42.616Z 1 020-039 125 125 125 227 227 227 F F F
136222024-01-10T18:00:42.616Z 2 040-059 125 126 126 227 230 230 F F F <---
136232024-01-10T18:00:42.616Z 3 060-079 125 125 125 227 227 227 F F F
136242024-01-10T18:00:42.616Z 4 080-099 124 124 124 226 226 226 F F F
136252024-01-10T18:00:43.028Z 5 100-119 124 124 124 226 226 226 F F F
136262024-01-10T18:00:43.028Z 6 120-139 124 124 124 226 226 226 F F F
136272024-01-10T18:00:43.028Z 7 140-159 119 119 119 219 219 219 F F F
136282024-01-10T18:00:43.028Z 8 160-179 123 123 123 224 224 224 F F F
136292024-01-10T18:00:43.028Z 9 180-199 124 124 124 225 225 225 F F F
136302024-01-10T18:00:43.028Z 10 200-219 124 124 124 225 225 225 F F F
136312024-01-10T18:00:43.028Z 11 220-239 120 126 126 221 228 228 F F F <---
136322024-01-10T18:00:43.028Z 12 240-259 122 122 122 223 223 223 F T T <---
136332024-01-10T18:00:43.028Z 13 260-279 122 126 126 223 228 228 F T T <---
136342024-01-10T18:00:43.028Z 14 280-299 125 126 126 227 228 228 T F F <---
136352024-01-10T18:00:43.029Z 15 300-319 124 126 126 225 230 230 F F F <---
136362024-01-10T18:00:43.029Z 16 320-339 124 124 124 226 226 226 F T T <---
136372024-01-10T18:00:43.029Z 17 340-359 124 124 124 225 225 225 F F F
136382024-01-10T18:00:43.029Z 18 360-379 121 121 121 222 222 222 F F F
136392024-01-10T18:00:43.029Z 19 380-399 122 122 122 223 223 223 F F F
136402024-01-10T18:00:43.029Z 20 400-419 124 126 126 226 230 230 F F F <---
136412024-01-10T18:00:43.029Z 21 420-439 123 123 123 224 224 224 F F F
136422024-01-10T18:00:43.029Z 22 440-459 123 123 123 224 224 224 F F F
136432024-01-10T18:00:43.029Z 23 460-479 125 125 125 227 227 227 F F F
136442024-01-10T18:00:43.029Z 24 480-499 125 125 125 227 227 227 F F F
136452024-01-10T18:00:43.029Z 25 500-519 123 123 123 224 224 224 F F F
136462024-01-10T18:00:43.030Z 26 520-539 123 123 123 224 224 224 F T T <---
136472024-01-10T18:00:43.030Z 27 540-559 123 123 123 224 224 224 F T T <---
136482024-01-10T18:00:43.030Z 28 560-579 125 126 126 227 228 228 F F F <---
136492024-01-10T18:00:43.030Z 29 580-599 116 116 116 216 216 216 F F F
136502024-01-10T18:00:43.030ZMax gen: 126, Max flush: 230
136512024-01-10T18:00:43.031ZError: Difference in extent metadata found!
136522024-01-10T18:00:43.031ZA Difference in extent metadata is expected here
136532024-01-10T18:00:43.031ZOn loop 63
136542024-01-10T18:00:43.031Z
136552024-01-10T18:00:43.040ZVerifying data now
136562024-01-10T18:00:47.741ZNo change after verify
136572024-01-10T18:00:47.741ZLoop: 63 Downstairs dump after verify (and repair):
136582024-01-10T18:00:47.763ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136592024-01-10T18:00:47.763Z 0 000-019 119 119 119 219 219 219 F F F
136602024-01-10T18:00:47.763Z 1 020-039 125 125 125 227 227 227 F F F
136612024-01-10T18:00:47.763Z 2 040-059 126 126 126 230 230 230 F F F
136622024-01-10T18:00:47.763Z 3 060-079 125 125 125 227 227 227 F F F
136632024-01-10T18:00:47.763Z 4 080-099 124 124 124 226 226 226 F F F
136642024-01-10T18:00:47.763Z 5 100-119 124 124 124 226 226 226 F F F
136652024-01-10T18:00:47.767Z 6 120-139 124 124 124 226 226 226 F F F
136662024-01-10T18:00:47.767Z 7 140-159 119 119 119 219 219 219 F F F
136672024-01-10T18:00:47.767Z 8 160-179 123 123 123 224 224 224 F F F
136682024-01-10T18:00:47.767Z 9 180-199 124 124 124 225 225 225 F F F
136692024-01-10T18:00:47.767Z 10 200-219 124 124 124 225 225 225 F F F
136702024-01-10T18:00:47.767Z 11 220-239 126 126 126 228 228 228 F F F
136712024-01-10T18:00:47.768Z 12 240-259 127 127 127 231 231 231 F F F
136722024-01-10T18:00:47.768Z 13 260-279 127 127 127 231 231 231 F F F
136732024-01-10T18:00:47.768Z 14 280-299 126 126 126 228 228 228 F F F
136742024-01-10T18:00:47.768Z 15 300-319 126 126 126 230 230 230 F F F
136752024-01-10T18:00:47.768Z 16 320-339 127 127 127 231 231 231 F F F
136762024-01-10T18:00:47.768Z 17 340-359 124 124 124 225 225 225 F F F
136772024-01-10T18:00:47.768Z 18 360-379 121 121 121 222 222 222 F F F
136782024-01-10T18:00:47.768Z 19 380-399 122 122 122 223 223 223 F F F
136792024-01-10T18:00:47.768Z 20 400-419 126 126 126 230 230 230 F F F
136802024-01-10T18:00:47.768Z 21 420-439 123 123 123 224 224 224 F F F
136812024-01-10T18:00:47.768Z 22 440-459 123 123 123 224 224 224 F F F
136822024-01-10T18:00:47.768Z 23 460-479 125 125 125 227 227 227 F F F
136832024-01-10T18:00:47.769Z 24 480-499 125 125 125 227 227 227 F F F
136842024-01-10T18:00:47.769Z 25 500-519 123 123 123 224 224 224 F F F
136852024-01-10T18:00:47.769Z 26 520-539 127 127 127 231 231 231 F F F
136862024-01-10T18:00:47.769Z 27 540-559 127 127 127 231 231 231 F F F
136872024-01-10T18:00:47.769Z 28 560-579 126 126 126 228 228 228 F F F
136882024-01-10T18:00:47.769Z 29 580-599 116 116 116 216 216 216 F F F
136892024-01-10T18:00:47.769ZMax gen: 127, Max flush: 231
136902024-01-10T18:00:47.774Z
136912024-01-10T18:00:47.774ZBegin loop 64
136922024-01-10T18:00:47.774ZDownstairs to restart: 1
136932024-01-10T18:00:47.797Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.803131555Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670}
136942024-01-10T18:00:47.797Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.803634523Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670}
136952024-01-10T18:00:47.805Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.803690238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670}
136962024-01-10T18:00:47.805Z{"msg":"Crucible 87bbabc0-cfb7-484a-8543-166dd7218e8a has session id: 68e2ad17-36ae-4c89-ae93-2e65a8f02f40","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.803790361Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
136972024-01-10T18:00:47.805Z{"msg":"Upstairs opts: Upstairs UUID: 87bbabc0-cfb7-484a-8543-166dd7218e8a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.803903978Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
136982024-01-10T18:00:47.805Z{"msg":"Crucible stats registered with UUID: 87bbabc0-cfb7-484a-8543-166dd7218e8a","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.803966112Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"Crucible runtime is spawned
136992024-01-10T18:00:47.805Z68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137002024-01-10T18:00:47.805Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804088154Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670}
137012024-01-10T18:00:47.805Z{"msg":"87bbabc0-cfb7-484a-8543-166dd7218e8a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804330628Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137022024-01-10T18:00:47.805Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804378956Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"0","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137032024-01-10T18:00:47.805Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804404797Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"0","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137042024-01-10T18:00:47.805Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804444861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"1","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137052024-01-10T18:00:47.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804468881Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"1","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137062024-01-10T18:00:47.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804538936Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"2","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137072024-01-10T18:00:47.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804577418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"2","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137082024-01-10T18:00:47.806Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804632347Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137092024-01-10T18:00:47.806Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804704554Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137102024-01-10T18:00:47.806Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804774393Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137112024-01-10T18:00:47.806Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.804931205Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137122024-01-10T18:00:47.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.805029796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137132024-01-10T18:00:47.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.805076058Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137142024-01-10T18:00:47.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.806299081Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"2","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137152024-01-10T18:00:47.808Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.806436207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"2","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137162024-01-10T18:00:47.808Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.806549125Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"0","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137172024-01-10T18:00:47.808Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.806590703Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"0","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137182024-01-10T18:00:47.808Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.80714138Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"1","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137192024-01-10T18:00:47.808Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.807208309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"1","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137202024-01-10T18:00:47.808Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.807242674Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"2","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137212024-01-10T18:00:47.808Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.807799118Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"0","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137222024-01-10T18:00:47.808Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.808357532Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"1","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137232024-01-10T18:00:47.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.808434557Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"2","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137242024-01-10T18:00:47.808Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.808460447Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"2","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137252024-01-10T18:00:47.809Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.808482966Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137262024-01-10T18:00:47.809Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.808707718Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"0","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137272024-01-10T18:00:47.809Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.808768255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"0","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137282024-01-10T18:00:47.809Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.808793792Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137292024-01-10T18:00:47.809Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.809238621Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"1","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137302024-01-10T18:00:47.813Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.809317664Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"1","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137312024-01-10T18:00:47.813Z{"msg":"[0]R flush_numbers[0..12]: [219, 227, 230, 227, 226, 226, 226, 219, 224, 225, 225, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.809392065Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137322024-01-10T18:00:47.813Z{"msg":"[0]R generation[0..12]: [119, 125, 126, 125, 124, 124, 124, 119, 123, 124, 124, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.809466789Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137332024-01-10T18:00:47.813Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.809541465Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137342024-01-10T18:00:47.813Z{"msg":"[1]R flush_numbers[0..12]: [219, 227, 230, 227, 226, 226, 226, 219, 224, 225, 225, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.809616148Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137352024-01-10T18:00:47.815Z{"msg":"[1]R generation[0..12]: [119, 125, 126, 125, 124, 124, 124, 119, 123, 124, 124, 126]","v":0Wait for a query_work_queue command to finish before sending IO
137362024-01-10T18:00:47.815Z,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.809785223Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137372024-01-10T18:00:47.815Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.809859206Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137382024-01-10T18:00:47.815Z{"msg":"[2]R flush_numbers[0..12]: [219, 227, 230, 227, 226, 226, 226, 219, 224, 225, 225, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.809933998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137392024-01-10T18:00:47.815Z{"msg":"[2]R generation[0..12]: [119, 125, 126, 125, 124, 124, 124, 119, 123, 124, 124, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810006406Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137402024-01-10T18:00:47.815Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810051374Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137412024-01-10T18:00:47.815Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810144715Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137422024-01-10T18:00:47.815Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810189934Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137432024-01-10T18:00:47.815Z{"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810256291Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137442024-01-10T18:00:47.815Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810295127Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"":"downstairs","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137452024-01-10T18:00:47.816Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810337656Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137462024-01-10T18:00:47.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810358962Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"0","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137472024-01-10T18:00:47.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810466135Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"1","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137482024-01-10T18:00:47.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810502006Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"client":"2","":"downstairs","session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137492024-01-10T18:00:47.816Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810525407Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137502024-01-10T18:00:47.816Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.81054528Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137512024-01-10T18:00:47.816Z{"msg":"87bbabc0-cfb7-484a-8543-166dd7218e8a is now active with session: 68e2ad17-36ae-4c89-ae93-2e65a8f02f40","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810586414Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137522024-01-10T18:00:47.816Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810607239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670,"session_id":"68e2ad17-36ae-4c89-ae93-2e65a8f02f40"}
137532024-01-10T18:00:47.816Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:47.810627459Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1670}
137542024-01-10T18:00:47.816ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
137552024-01-10T18:00:47.816ZLoading write count information from file "/tmp/test_repair_verify.data"
137562024-01-10T18:00:47.816ZRun Repair workload
137572024-01-10T18:00:47.817Z01/30 Write block 574 len 40960 data: 9 10 11 9 8 9 10 10 10 11
137582024-01-10T18:00:47.817Z02/30 Write block 290 len 24576 data: 14 15 17 13 11 12
137592024-01-10T18:00:47.817Z03/30 Read block 284 len 4096
137602024-01-10T18:00:47.817Z04/30 Read block 36 len 36864
137612024-01-10T18:00:47.820Z05/30 Write block 62 len 20480 data: 8 8 9 11 13
137622024-01-10T18:00:47.820Z06/30 Write block 39 len 40960 data: 7 6 9 10 9 9 9 8 6 11
137632024-01-10T18:00:47.820Z07/30 Write block 527 len 4096 data: 11
137642024-01-10T18:00:47.821Z08/30 Read block 134 len 40960
137652024-01-10T18:00:47.824Z09/30 Read block 72 len 24576
137662024-01-10T18:00:47.827Z10/30 Write block 49 len 8192 data: 13 13
137672024-01-10T18:00:47.827Z11/30 Write block 456 len 36864 data: 10 11 9 9 9 8 8 9 9
137682024-01-10T18:00:47.827Z12/30 Write block 212 len 20480 data: 10 10 10 8 11
137692024-01-10T18:00:47.830Z13/30 Read block 563 len 4096
137702024-01-10T18:00:47.830Z14/30 Flush
137712024-01-10T18:00:47.836Z15/30 Flush
137722024-01-10T18:00:47.839Z16/30 Write block 455 len 24576 data: 10 11 12 10 10 10
137732024-01-10T18:00:47.839Z17/30 Write block 509 len 4096 data: 11
137742024-01-10T18:00:47.839Z18/30 Write block 83 len 4096 data: 12
137752024-01-10T18:00:47.839Z19/30 Read block 120 len 28672
137762024-01-10T18:00:47.839Z20/30 Read block 548 len 20480
137772024-01-10T18:00:47.842Z21/30 Write block 463 len 40960 data: 10 10 8 10 11 11 7 7 4 3
137782024-01-10T18:00:47.845Z22/30 Write block 433 len 40960 data: 7 7 6 7 8 8 9 8 11 8
137792024-01-10T18:00:47.845Z23/30 Write block 540 len 8192 data: 13 13
137802024-01-10T18:00:47.845Z24/30 Flush
137812024-01-10T18:00:47.849Z25/30 Write block 118 len 24576 data: 8 10 10 9 9 7
137822024-01-10T18:00:47.853Z26/30 Read block 253 len 4096
137832024-01-10T18:00:47.853Z27/30 Read block 523 len 16384
137842024-01-10T18:00:47.856Z28/30 Read block 176 len 32768
137852024-01-10T18:00:47.856Z29/30 Read block 515 len 32768
137862024-01-10T18:00:47.859Z30/30 Write block 106 len 16384 data: 6 5 6 9
137872024-01-10T18:00:47.862Z----------------------------------------------------------------
137882024-01-10T18:00:47.862Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
137892024-01-10T18:00:47.862ZStates: Active Active Active
137902024-01-10T18:00:47.862ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
137912024-01-10T18:00:47.862Z 1 Acked 1000 Write 10 Done Done Done false
137922024-01-10T18:00:47.862Z 2 Acked 1001 Write 6 Done Sent Done false
137932024-01-10T18:00:47.862Z 3 Acked 1002 Read 1 Done Sent Done false
137942024-01-10T18:00:47.862Z 4 Acked 1003 Read 9 Done Sent Done false
137952024-01-10T18:00:47.862Z 5 Acked 1004 Write 5 Done Sent Done false
137962024-01-10T18:00:47.862Z 6 Acked 1005 Write 10 Done Sent Done false
137972024-01-10T18:00:47.862Z 7 Acked 1006 Write 1 Done Sent Done false
137982024-01-10T18:00:47.862Z 8 Acked 1007 Read 10 Done Sent Done false
137992024-01-10T18:00:47.862Z 9 Acked 1008 Read 6 Done Sent Done false
138002024-01-10T18:00:47.862Z 10 Acked 1009 Write 2 Done Sent Done false
138012024-01-10T18:00:47.863Z 11 Acked 1010 Write 9 Done Sent Done false
138022024-01-10T18:00:47.863Z 12 Acked 1011 Write 5 Done Sent Done false
138032024-01-10T18:00:47.863Z 13 Acked 1012 Read 1 Done Sent Done false
138042024-01-10T18:00:47.863Z 14 Acked 1013 Flush 0 Done Sent Done false
138052024-01-10T18:00:47.863Z 15 Acked 1014 Flush 0 Done Sent Done false
138062024-01-10T18:00:47.863Z 16 Acked 1015 Write 6 Done Sent Done false
138072024-01-10T18:00:47.863Z 17 Acked 1016 Write 1 Done Sent Done false
138082024-01-10T18:00:47.863Z 18 Acked 1017 Write 1 Done Sent Done false
138092024-01-10T18:00:47.863Z 19 Acked 1018 Read 7 Done Sent Done false
138102024-01-10T18:00:47.863Z 20 Acked 1019 Read 5 Done Sent Done false
138112024-01-10T18:00:47.863Z 21 Acked 1020 Write 10 Done Sent Done false
138122024-01-10T18:00:47.863Z 22 Acked 1021 Write 10 Done Sent Done false
138132024-01-10T18:00:47.863Z 23 Acked 1022 Write 2 Done Sent Done false
138142024-01-10T18:00:47.863Z 24 Acked 1023 Flush 0 Done Sent Done false
138152024-01-10T18:00:47.863Z 25 Acked 1024 Write 6 Done Sent Done false
138162024-01-10T18:00:47.863Z 26 Acked 1025 Read 1 Done Sent Done false
138172024-01-10T18:00:47.863Z 27 Acked 1026 Read 4 Done Sent Done false
138182024-01-10T18:00:47.863Z 28 Acked 1027 Read 8 Done Sent Done false
138192024-01-10T18:00:47.863Z 29 Acked 1028 Read 8 Done Sent Done false
138202024-01-10T18:00:47.863Z 30 Acked 1029 Write 4 Sent Sent Sent false
138212024-01-10T18:00:47.863Z STATES DS:0 DS:1 DS:2 TOTAL
138222024-01-10T18:00:47.863Z New 0 0 0 0
138232024-01-10T18:00:47.863Z Sent 1 29 1 31
138242024-01-10T18:00:47.863Z Done 29 1 29 59
138252024-01-10T18:00:47.864Z Skipped 0 0 0 0
138262024-01-10T18:00:47.864Z Error 0 0 0 0
138272024-01-10T18:00:47.864ZLast Flush: 1023 0 1023
138282024-01-10T18:00:47.864ZDownstairs last five completed:
138292024-01-10T18:00:47.864ZUpstairs last five completed: 30 29 28 27 26
138302024-01-10T18:00:47.864ZWrote out file "/tmp/test_repair_verify.data"
138312024-01-10T18:00:47.867Z
138322024-01-10T18:00:47.872ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138332024-01-10T18:00:47.895ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138342024-01-10T18:00:47.895Z 0 000-019 119 119 119 219 219 219 F F F
138352024-01-10T18:00:47.895Z 1 020-039 128 125 128 232 227 232 F F F <---
138362024-01-10T18:00:47.895Z 2 040-059 128 126 128 232 230 232 F F F <---
138372024-01-10T18:00:47.895Z 3 060-079 128 125 128 232 227 232 F F F <---
138382024-01-10T18:00:47.898Z 4 080-099 128 124 128 234 226 234 F F F <---
138392024-01-10T18:00:47.898Z 5 100-119 124 124 124 226 226 226 T F T <---
138402024-01-10T18:00:47.898Z 6 120-139 124 124 124 226 226 226 T F T <---
138412024-01-10T18:00:47.898Z 7 140-159 119 119 119 219 219 219 F F F
138422024-01-10T18:00:47.899Z 8 160-179 123 123 123 224 224 224 F F F
138432024-01-10T18:00:47.899Z 9 180-199 124 124 124 225 225 225 F F F
138442024-01-10T18:00:47.899Z 10 200-219 128 124 128 232 225 232 F F F <---
138452024-01-10T18:00:47.899Z 11 220-239 126 126 126 228 228 228 F F F
138462024-01-10T18:00:47.899Z 12 240-259 127 127 127 231 231 231 F F F
138472024-01-10T18:00:47.899Z 13 260-279 127 127 127 231 231 231 F F F
138482024-01-10T18:00:47.899Z 14 280-299 128 126 128 232 228 232 F F F <---
138492024-01-10T18:00:47.899Z 15 300-319 126 126 126 230 230 230 F F F
138502024-01-10T18:00:47.899Z 16 320-339 127 127 127 231 231 231 F F F
138512024-01-10T18:00:47.899Z 17 340-359 124 124 124 225 225 225 F F F
138522024-01-10T18:00:47.899Z 18 360-379 121 121 121 222 222 222 F F F
138532024-01-10T18:00:47.899Z 19 380-399 122 122 122 223 223 223 F F F
138542024-01-10T18:00:47.899Z 20 400-419 126 126 126 230 230 230 F F F
138552024-01-10T18:00:47.899Z 21 420-439 128 123 128 234 224 234 F F F <---
138562024-01-10T18:00:47.899Z 22 440-459 128 123 128 234 224 234 F F F <---
138572024-01-10T18:00:47.899Z 23 460-479 128 125 128 234 227 234 F F F <---
138582024-01-10T18:00:47.899Z 24 480-499 125 125 125 227 227 227 F F F
138592024-01-10T18:00:47.899Z 25 500-519 128 123 128 234 224 234 F F F <---
138602024-01-10T18:00:47.899Z 26 520-539 128 127 128 232 231 232 F F F <---
138612024-01-10T18:00:47.899Z 27 540-559 128 127 128 234 231 234 F F F <---
138622024-01-10T18:00:47.899Z 28 560-579 128 126 128 232 228 232 F T F <---
138632024-01-10T18:00:47.900Z 29 580-599 128 116 128 232 216 232 F T F <---
138642024-01-10T18:00:47.900ZMax gen: 128, Max flush: 234
138652024-01-10T18:00:47.900ZError: Difference in extent metadata found!
138662024-01-10T18:00:47.903ZA Difference in extent metadata is expected here
138672024-01-10T18:00:47.903ZOn loop 64
138682024-01-10T18:00:47.903Z
138692024-01-10T18:00:47.906ZVerifying data now
138702024-01-10T18:00:53.397ZNo change after verify
138712024-01-10T18:00:53.397ZLoop: 64 Downstairs dump after verify (and repair):
138722024-01-10T18:00:53.419ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138732024-01-10T18:00:53.419Z 0 000-019 119 119 119 219 219 219 F F F
138742024-01-10T18:00:53.419Z 1 020-039 128 128 128 232 232 232 F F F
138752024-01-10T18:00:53.419Z 2 040-059 128 128 128 232 232 232 F F F
138762024-01-10T18:00:53.419Z 3 060-079 128 128 128 232 232 232 F F F
138772024-01-10T18:00:53.419Z 4 080-099 128 128 128 234 234 234 F F F
138782024-01-10T18:00:53.419Z 5 100-119 129 129 129 235 235 235 F F F
138792024-01-10T18:00:53.420Z 6 120-139 129 129 129 235 235 235 F F F
138802024-01-10T18:00:53.420Z 7 140-159 119 119 119 219 219 219 F F F
138812024-01-10T18:00:53.420Z 8 160-179 123 123 123 224 224 224 F F F
138822024-01-10T18:00:53.420Z 9 180-199 124 124 124 225 225 225 F F F
138832024-01-10T18:00:53.423Z 10 200-219 128 128 128 232 232 232 F F F
138842024-01-10T18:00:53.423Z 11 220-239 126 126 126 228 228 228 F F F
138852024-01-10T18:00:53.424Z 12 240-259 127 127 127 231 231 231 F F F
138862024-01-10T18:00:53.424Z 13 260-279 127 127 127 231 231 231 F F F
138872024-01-10T18:00:53.424Z 14 280-299 128 128 128 232 232 232 F F F
138882024-01-10T18:00:53.424Z 15 300-319 126 126 126 230 230 230 F F F
138892024-01-10T18:00:53.424Z 16 320-339 127 127 127 231 231 231 F F F
138902024-01-10T18:00:53.424Z 17 340-359 124 124 124 225 225 225 F F F
138912024-01-10T18:00:53.424Z 18 360-379 121 121 121 222 222 222 F F F
138922024-01-10T18:00:53.424Z 19 380-399 122 122 122 223 223 223 F F F
138932024-01-10T18:00:53.425Z 20 400-419 126 126 126 230 230 230 F F F
138942024-01-10T18:00:53.425Z 21 420-439 128 128 128 234 234 234 F F F
138952024-01-10T18:00:53.425Z 22 440-459 128 128 128 234 234 234 F F F
138962024-01-10T18:00:53.425Z 23 460-479 128 128 128 234 234 234 F F F
138972024-01-10T18:00:53.425Z 24 480-499 125 125 125 227 227 227 F F F
138982024-01-10T18:00:53.425Z 25 500-519 128 128 128 234 234 234 F F F
138992024-01-10T18:00:53.425Z 26 520-539 128 128 128 232 232 232 F F F
139002024-01-10T18:00:53.425Z 27 540-559 128 128 128 234 234 234 F F F
139012024-01-10T18:00:53.425Z 28 560-579 128 128 128 232 232 232 F F F
139022024-01-10T18:00:53.425Z 29 580-599 128 128 128 232 232 232 F F F
139032024-01-10T18:00:53.425ZMax gen: 129, Max flush: 235
139042024-01-10T18:00:53.428Z
139052024-01-10T18:00:53.428ZBegin loop 65
139062024-01-10T18:00:53.428ZDownstairs to restart: 1
139072024-01-10T18:00:53.454Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.459674298Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678}
139082024-01-10T18:00:53.454Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.460210549Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678}
139092024-01-10T18:00:53.454Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.460240882Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678}
139102024-01-10T18:00:53.458Z{"msg":"Crucible d012a003-ce0d-4c83-a9ce-add1e755c2ea has session id: 91e1c646-9d46-46b2-8eba-626c5fdaa0d0","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.460302551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139112024-01-10T18:00:53.458Z{Crucible runtime is spawned
139122024-01-10T18:00:53.459Z"msg":"Upstairs opts: Upstairs UUID: d012a003-ce0d-4c83-a9ce-add1e755c2ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.460473672Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139132024-01-10T18:00:53.459Z{"msg":"Crucible stats registered with UUID: d012a003-ce0d-4c83-a9ce-add1e755c2ea","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.460609552Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139142024-01-10T18:00:53.459Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.460651696Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678}
139152024-01-10T18:00:53.459Z{"msg":"d012a003-ce0d-4c83-a9ce-add1e755c2ea active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.460926919Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139162024-01-10T18:00:53.459Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461092441Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"0","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139172024-01-10T18:00:53.459Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461140442Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"0","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139182024-01-10T18:00:53.459Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461180459Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"1","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139192024-01-10T18:00:53.459Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461233805Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"1","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139202024-01-10T18:00:53.459Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461276625Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"2","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139212024-01-10T18:00:53.459Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461319469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"2","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139222024-01-10T18:00:53.460Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461390119Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139232024-01-10T18:00:53.460Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461483491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139242024-01-10T18:00:53.460Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461558517Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139252024-01-10T18:00:53.460Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461673412Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139262024-01-10T18:00:53.460Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461761356Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139272024-01-10T18:00:53.460Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.461837675Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139282024-01-10T18:00:53.461Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.462754152Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"0","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139292024-01-10T18:00:53.461Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.462818076Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"0","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139302024-01-10T18:00:53.461Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.463064966Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"2","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139312024-01-10T18:00:53.461Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.463121917Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"2","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139322024-01-10T18:00:53.461Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.46379579Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"0","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139332024-01-10T18:00:53.465Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.463983193Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"2","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139342024-01-10T18:00:53.465Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.46459752Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"0","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139352024-01-10T18:00:53.465Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.464659778Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"0","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139362024-01-10T18:00:53.466Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.464763583Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139372024-01-10T18:00:53.466Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.464832048Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"1","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139382024-01-10T18:00:53.466Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.464909958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"1","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139392024-01-10T18:00:53.466Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.464988406Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"2","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139402024-01-10T18:00:53.466Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.465060123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"2","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139412024-01-10T18:00:53.466Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.465089554Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139422024-01-10T18:00:53.466Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.465671667Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"1","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139432024-01-10T18:00:53.467Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466320841Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"1","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139442024-01-10T18:00:53.467Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466389441Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"1","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139452024-01-10T18:00:53.467Z{"msg":"[0]R flush_numbers[0..12]: [219, 232, 232, 232, 234, 235, 235, 219, 224, 225, 232, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466431107Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139462024-01-10T18:00:53.467Z{"msg":"[0]R generation[0..12]: [119, 128, 128, 128, 128, 129, 129, 119, 123, 124, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466495098Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139472024-01-10T18:00:53.467Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466531137Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139482024-01-10T18:00:53.467Z{"msg":"[1]R flush_numbers[0..12]: [219, 232, 232, 232, 234, 235, 235, 219, 224, 225, 232, 228]","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO
139492024-01-10T18:00:53.467Z":30,"time":"2024-01-10T18:00:53.466591741Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139502024-01-10T18:00:53.467Z{"msg":"[1]R generation[0..12]: [119, 128, 128, 128, 128, 129, 129, 119, 123, 124, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466627582Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139512024-01-10T18:00:53.467Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466684052Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139522024-01-10T18:00:53.468Z{"msg":"[2]R flush_numbers[0..12]: [219, 232, 232, 232, 234, 235, 235, 219, 224, 225, 232, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.46671621Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139532024-01-10T18:00:53.468Z{"msg":"[2]R generation[0..12]: [119, 128, 128, 128, 128, 129, 129, 119, 123, 124, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466774561Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139542024-01-10T18:00:53.468Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466808324Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139552024-01-10T18:00:53.468Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466864076Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139562024-01-10T18:00:53.468Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466894413Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139572024-01-10T18:00:53.468Z{"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466950367Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139582024-01-10T18:00:53.468Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.466980612Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"":"downstairs","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139592024-01-10T18:00:53.468Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.467044842Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139602024-01-10T18:00:53.468Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.467068687Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"0","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139612024-01-10T18:00:53.468Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.467097414Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"1","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139622024-01-10T18:00:53.468Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.46715253Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"client":"2","":"downstairs","session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139632024-01-10T18:00:53.468Z{"msg":"All required repair work is completed","v":0,"name"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
139642024-01-10T18:00:53.468Z:"crucible","level":30,"time":"2024-01-10T18:00:53.467260637Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139652024-01-10T18:00:53.468Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.467317219Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139662024-01-10T18:00:53.468Z{"msg":"d012a003-ce0d-4c83-a9ce-add1e755c2ea is now active with session: 91e1c646-9d46-46b2-8eba-626c5fdaa0d0","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.467348006Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139672024-01-10T18:00:53.469Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.467401423Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678,"session_id":"91e1c646-9d46-46b2-8eba-626c5fdaa0d0"}
139682024-01-10T18:00:53.469Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:00:53.467425846Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1678}
139692024-01-10T18:00:53.469ZLoading write count information from file "/tmp/test_repair_verify.data"
139702024-01-10T18:00:53.469ZRun Repair workload
139712024-01-10T18:00:53.469Z01/30 Write block 270 len 20480 data: 11 11 8 8 13
139722024-01-10T18:00:53.472Z02/30 Write block 435 len 20480 data: 7 8 9 9 10
139732024-01-10T18:00:53.472Z03/30 Read block 321 len 4096
139742024-01-10T18:00:53.473Z04/30 Read block 133 len 36864
139752024-01-10T18:00:53.473Z05/30 Write block 287 len 12288 data: 10 9 12
139762024-01-10T18:00:53.473Z06/30 Read block 480 len 12288
139772024-01-10T18:00:53.476Z07/30 Write block 60 len 36864 data: 11 9 9 9 10 12 14 11 10
139782024-01-10T18:00:53.476Z08/30 Read block 446 len 36864
139792024-01-10T18:00:53.479Z09/30 Write block 342 len 24576 data: 13 13 12 12 12 13
139802024-01-10T18:00:53.480Z10/30 Read block 116 len 20480
139812024-01-10T18:00:53.482Z11/30 Write block 166 len 24576 data: 15 14 12 12 10 7
139822024-01-10T18:00:53.485Z12/30 Write block 318 len 36864 data: 8 8 8 9 8 10 12 12 10
139832024-01-10T18:00:53.485Z13/30 Read block 199 len 32768
139842024-01-10T18:00:53.488Z14/30 Read block 323 len 20480
139852024-01-10T18:00:53.491Z15/30 Flush
139862024-01-10T18:00:53.494Z16/30 Write block 478 len 12288 data: 3 5 6
139872024-01-10T18:00:53.497Z17/30 Write block 177 len 24576 data: 8 8 8 10 9 9
139882024-01-10T18:00:53.497Z18/30 Read block 49 len 28672
139892024-01-10T18:00:53.500Z19/30 Write block 520 len 40960 data: 11 12 13 8 7 10 12 12 11 12
139902024-01-10T18:00:53.500Z20/30 Read block 262 len 40960
139912024-01-10T18:00:53.505Z21/30 Write block 572 len 36864 data: 6 7 10 11 12 10 9 10 11
139922024-01-10T18:00:53.509Z22/30 Read block 70 len 36864
139932024-01-10T18:00:53.512Z23/30 Read block 146 len 12288
139942024-01-10T18:00:53.512Z24/30 Write block 215 len 16384 data: 9 12 11 9
139952024-01-10T18:00:53.512Z25/30 Read block 85 len 36864
139962024-01-10T18:00:53.515Z26/30 Write block 241 len 12288 data: 8 10 11
139972024-01-10T18:00:53.518Z27/30 Read block 488 len 20480
139982024-01-10T18:00:53.521Z28/30 Write block 243 len 32768 data: 12 16 15 10 9 10 10 8
139992024-01-10T18:00:53.521Z29/30 Read block 287 len 24576
140002024-01-10T18:00:53.525Z30/30 Read block 183 len 4096
140012024-01-10T18:00:53.525Z----------------------------------------------------------------
140022024-01-10T18:00:53.525Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
140032024-01-10T18:00:53.527ZStates: Active Active Active
140042024-01-10T18:00:53.527ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140052024-01-10T18:00:53.527Z 1 Acked 1000 Write 5 Done Done Done false
140062024-01-10T18:00:53.527Z 2 Acked 1001 Write 5 Done Done Done false
140072024-01-10T18:00:53.527Z 3 Acked 1002 Read 1 Done Done Done false
140082024-01-10T18:00:53.527Z 4 Acked 1003 Read 9 Done Sent Done false
140092024-01-10T18:00:53.527Z 5 Acked 1004 Write 3 Done Sent Done false
140102024-01-10T18:00:53.527Z 6 Acked 1005 Read 3 Done Sent Done false
140112024-01-10T18:00:53.528Z 7 Acked 1006 Write 9 Done Sent Done false
140122024-01-10T18:00:53.528Z 8 Acked 1007 Read 9 Done Sent Done false
140132024-01-10T18:00:53.528Z 9 Acked 1008 Write 6 Done Sent Done false
140142024-01-10T18:00:53.528Z 10 Acked 1009 Read 5 Done Sent Done false
140152024-01-10T18:00:53.528Z 11 Acked 1010 Write 6 Done Sent Done false
140162024-01-10T18:00:53.528Z 12 Acked 1011 Write 9 Done Sent Done false
140172024-01-10T18:00:53.528Z 13 Acked 1012 Read 8 Done Sent Done false
140182024-01-10T18:00:53.528Z 14 Acked 1013 Read 5 Done Sent Done false
140192024-01-10T18:00:53.528Z 15 Acked 1014 Flush 0 Done Sent Done false
140202024-01-10T18:00:53.528Z 16 Acked 1015 Write 3 Done Sent Done false
140212024-01-10T18:00:53.528Z 17 Acked 1016 Write 6 Done Sent Done false
140222024-01-10T18:00:53.528Z 18 Acked 1017 Read 7 Done Sent Done false
140232024-01-10T18:00:53.528Z 19 Acked 1018 Write 10 Done Sent Done false
140242024-01-10T18:00:53.528Z 20 Acked 1019 Read 10 Done Sent Done false
140252024-01-10T18:00:53.528Z 21 Acked 1020 Write 9 Done Sent Done false
140262024-01-10T18:00:53.528Z 22 Acked 1021 Read 9 Done Sent Done false
140272024-01-10T18:00:53.528Z 23 Acked 1022 Read 3 Done Sent Done false
140282024-01-10T18:00:53.528Z 24 Acked 1023 Write 4 Done Sent Done false
140292024-01-10T18:00:53.528Z 25 Acked 1024 Read 9 Done Sent Done false
140302024-01-10T18:00:53.528Z 26 Acked 1025 Write 3 Done Sent Done false
140312024-01-10T18:00:53.528Z 27 Acked 1026 Read 5 Done Sent Done false
140322024-01-10T18:00:53.528Z 28 Acked 1027 Write 8 Done Sent Done false
140332024-01-10T18:00:53.528Z 29 Acked 1028 Read 6 Done Sent Done false
140342024-01-10T18:00:53.528Z 30 Acked 1029 Read 1 Done Sent Done false
140352024-01-10T18:00:53.529Z STATES DS:0 DS:1 DS:2 TOTAL
140362024-01-10T18:00:53.529Z New 0 0 0 0
140372024-01-10T18:00:53.529Z Sent 0 27 0 27
140382024-01-10T18:00:53.529Z Done 30 3 30 63
140392024-01-10T18:00:53.529Z Skipped 0 0 0 0
140402024-01-10T18:00:53.529Z Error 0 0 0 0
140412024-01-10T18:00:53.529ZLast Flush: 1014 0 1014
140422024-01-10T18:00:53.529ZDownstairs last five completed:
140432024-01-10T18:00:53.529ZUpstairs last five completed: 30 29 28 27 26
140442024-01-10T18:00:53.529ZWrote out file "/tmp/test_repair_verify.data"
140452024-01-10T18:00:53.532Z
140462024-01-10T18:00:53.537ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
140472024-01-10T18:00:53.565ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140482024-01-10T18:00:53.565Z 0 000-019 119 119 119 219 219 219 F F F
140492024-01-10T18:00:53.565Z 1 020-039 128 128 128 232 232 232 F F F
140502024-01-10T18:00:53.566Z 2 040-059 128 128 128 232 232 232 F F F
140512024-01-10T18:00:53.566Z 3 060-079 130 128 130 236 232 236 F F F <---
140522024-01-10T18:00:53.566Z 4 080-099 128 128 128 234 234 234 F F F
140532024-01-10T18:00:53.566Z 5 100-119 129 129 129 235 235 235 F F F
140542024-01-10T18:00:53.579Z 6 120-139 129 129 129 235 235 235 F F F
140552024-01-10T18:00:53.579Z 7 140-159 119 119 119 219 219 219 F F F
140562024-01-10T18:00:53.579Z 8 160-179 130 123 130 236 224 236 T F T <---
140572024-01-10T18:00:53.579Z 9 180-199 124 124 124 225 225 225 T F T <---
140582024-01-10T18:00:53.579Z 10 200-219 128 128 128 232 232 232 T F T <---
140592024-01-10T18:00:53.579Z 11 220-239 126 126 126 228 228 228 F F F
140602024-01-10T18:00:53.579Z 12 240-259 127 127 127 231 231 231 T F T <---
140612024-01-10T18:00:53.579Z 13 260-279 130 127 130 236 231 236 F T F <---
140622024-01-10T18:00:53.580Z 14 280-299 130 128 130 236 232 236 F F F <---
140632024-01-10T18:00:53.580Z 15 300-319 130 126 130 236 230 236 F F F <---
140642024-01-10T18:00:53.580Z 16 320-339 130 127 130 236 231 236 F F F <---
140652024-01-10T18:00:53.580Z 17 340-359 130 124 130 236 225 236 F F F <---
140662024-01-10T18:00:53.580Z 18 360-379 121 121 121 222 222 222 F F F
140672024-01-10T18:00:53.580Z 19 380-399 122 122 122 223 223 223 F F F
140682024-01-10T18:00:53.580Z 20 400-419 126 126 126 230 230 230 F F F
140692024-01-10T18:00:53.580Z 21 420-439 130 128 130 236 234 236 F T F <---
140702024-01-10T18:00:53.580Z 22 440-459 128 128 128 234 234 234 F F F
140712024-01-10T18:00:53.580Z 23 460-479 128 128 128 234 234 234 T F T <---
140722024-01-10T18:00:53.580Z 24 480-499 125 125 125 227 227 227 T F T <---
140732024-01-10T18:00:53.580Z 25 500-519 128 128 128 234 234 234 F F F
140742024-01-10T18:00:53.581Z 26 520-539 128 128 128 232 232 232 T F T <---
140752024-01-10T18:00:53.581Z 27 540-559 128 128 128 234 234 234 F F F
140762024-01-10T18:00:53.581Z 28 560-579 128 128 128 232 232 232 T F T <---
140772024-01-10T18:00:53.581Z 29 580-599 128 128 128 232 232 232 T F T <---
140782024-01-10T18:00:53.582ZMax gen: 130, Max flush: 236
140792024-01-10T18:00:53.583ZError: Difference in extent metadata found!
140802024-01-10T18:00:53.583ZA Difference in extent metadata is expected here
140812024-01-10T18:00:53.583ZOn loop 65
140822024-01-10T18:00:53.583Z
140832024-01-10T18:00:53.583ZVerifying data now
140842024-01-10T18:01:00.092ZNo change after verify
140852024-01-10T18:01:00.092ZLoop: 65 Downstairs dump after verify (and repair):
140862024-01-10T18:01:00.113ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140872024-01-10T18:01:00.113Z 0 000-019 119 119 119 219 219 219 F F F
140882024-01-10T18:01:00.113Z 1 020-039 128 128 128 232 232 232 F F F
140892024-01-10T18:01:00.113Z 2 040-059 128 128 128 232 232 232 F F F
140902024-01-10T18:01:00.113Z 3 060-079 130 130 130 236 236 236 F F F
140912024-01-10T18:01:00.113Z 4 080-099 128 128 128 234 234 234 F F F
140922024-01-10T18:01:00.113Z 5 100-119 129 129 129 235 235 235 F F F
140932024-01-10T18:01:00.114Z 6 120-139 129 129 129 235 235 235 F F F
140942024-01-10T18:01:00.117Z 7 140-159 119 119 119 219 219 219 F F F
140952024-01-10T18:01:00.117Z 8 160-179 131 131 131 237 237 237 F F F
140962024-01-10T18:01:00.117Z 9 180-199 131 131 131 237 237 237 F F F
140972024-01-10T18:01:00.117Z 10 200-219 131 131 131 237 237 237 F F F
140982024-01-10T18:01:00.117Z 11 220-239 126 126 126 228 228 228 F F F
140992024-01-10T18:01:00.117Z 12 240-259 131 131 131 237 237 237 F F F
141002024-01-10T18:01:00.117Z 13 260-279 130 130 130 236 236 236 F F F
141012024-01-10T18:01:00.117Z 14 280-299 130 130 130 236 236 236 F F F
141022024-01-10T18:01:00.117Z 15 300-319 130 130 130 236 236 236 F F F
141032024-01-10T18:01:00.117Z 16 320-339 130 130 130 236 236 236 F F F
141042024-01-10T18:01:00.117Z 17 340-359 130 130 130 236 236 236 F F F
141052024-01-10T18:01:00.117Z 18 360-379 121 121 121 222 222 222 F F F
141062024-01-10T18:01:00.117Z 19 380-399 122 122 122 223 223 223 F F F
141072024-01-10T18:01:00.117Z 20 400-419 126 126 126 230 230 230 F F F
141082024-01-10T18:01:00.117Z 21 420-439 130 130 130 236 236 236 F F F
141092024-01-10T18:01:00.117Z 22 440-459 128 128 128 234 234 234 F F F
141102024-01-10T18:01:00.117Z 23 460-479 131 131 131 237 237 237 F F F
141112024-01-10T18:01:00.117Z 24 480-499 131 131 131 237 237 237 F F F
141122024-01-10T18:01:00.118Z 25 500-519 128 128 128 234 234 234 F F F
141132024-01-10T18:01:00.118Z 26 520-539 131 131 131 237 237 237 F F F
141142024-01-10T18:01:00.118Z 27 540-559 128 128 128 234 234 234 F F F
141152024-01-10T18:01:00.118Z 28 560-579 131 131 131 237 237 237 F F F
141162024-01-10T18:01:00.118Z 29 580-599 131 131 131 237 237 237 F F F
141172024-01-10T18:01:00.118ZMax gen: 131, Max flush: 237
141182024-01-10T18:01:00.121Z
141192024-01-10T18:01:00.121ZBegin loop 66
141202024-01-10T18:01:00.121ZDownstairs to restart: 0
141212024-01-10T18:01:00.148Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.153394039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686}
141222024-01-10T18:01:00.148Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.153891217Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686}
141232024-01-10T18:01:00.152Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.153960893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686}
141242024-01-10T18:01:00.152Z{"msg":"Crucible runtime is spawned
141252024-01-10T18:01:00.152ZCrucible 1bebc3fc-6e70-4b26-8b3c-7364d3b4bca8 has session id: a303c634-565f-414d-9945-e6b4d493320a","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154141444Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141262024-01-10T18:01:00.153Z{"msg":"Upstairs opts: Upstairs UUID: 1bebc3fc-6e70-4b26-8b3c-7364d3b4bca8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154204922Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141272024-01-10T18:01:00.153Z{"msg":"Crucible stats registered with UUID: 1bebc3fc-6e70-4b26-8b3c-7364d3b4bca8","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154276815Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141282024-01-10T18:01:00.153Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154318251Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686}
141292024-01-10T18:01:00.153Z{"msg":"1bebc3fc-6e70-4b26-8b3c-7364d3b4bca8 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154381333Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141302024-01-10T18:01:00.153Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154452011Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"0","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141312024-01-10T18:01:00.153Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154525236Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"0","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141322024-01-10T18:01:00.153Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154569448Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"1","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141332024-01-10T18:01:00.153Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154662741Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"1","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141342024-01-10T18:01:00.153Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154709975Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"2","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141352024-01-10T18:01:00.153Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.154748547Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"2","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141362024-01-10T18:01:00.154Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.15484619Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141372024-01-10T18:01:00.154Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.155004844Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141382024-01-10T18:01:00.154Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.155083397Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141392024-01-10T18:01:00.154Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.155159242Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141402024-01-10T18:01:00.154Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.155188083Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141412024-01-10T18:01:00.154Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.155245543Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141422024-01-10T18:01:00.154Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.156264186Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"2","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141432024-01-10T18:01:00.154Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.156303589Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"2","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141442024-01-10T18:01:00.154Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.156460147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"1","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141452024-01-10T18:01:00.154Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.156545015Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"1","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141462024-01-10T18:01:00.155Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.157333507Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"2","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141472024-01-10T18:01:00.155Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.157565663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"1","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141482024-01-10T18:01:00.155Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.15771268Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"0","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141492024-01-10T18:01:00.160Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.157751015Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"0","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141502024-01-10T18:01:00.160Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.158225627Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"2","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141512024-01-10T18:01:00.160Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.158307469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"2","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141522024-01-10T18:01:00.160Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.158354027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141532024-01-10T18:01:00.160Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.158758943Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"1","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141542024-01-10T18:01:00.160Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.158823239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"1","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141552024-01-10T18:01:00.160Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.158866685Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141562024-01-10T18:01:00.162Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.158924922Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"0","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141572024-01-10T18:01:00.162Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.159645295Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"0","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141582024-01-10T18:01:00.162Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.159712008Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"0","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141592024-01-10T18:01:00.162Z{"msg":"[0]R flush_numbers[0..12]: [219, 232, 232, 236, 234, 235, 235, 219, 237, 237, 237, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.159740446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141602024-01-10T18:01:00.162Z{"msg":"[0]R generation[0..12]: [119, 128, 128, 130, 128, 129, 129, 119, 131, 131, 131, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.159783972Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141612024-01-10T18:01:00.162Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.159809834Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141622024-01-10T18:01:00.162Z{"msg":"[1]R flush_numbers[0..12]: [219, 232, 232, 236, 234, 235, 235, 219, 237, 237, 237, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.159851436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141632024-01-10T18:01:00.162Z{"msg":"[1]R generation[0..12]: [119, 128, 128, 130, 128, 129, 129, 119, 131, 131, 131, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.159876324Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141642024-01-10T18:01:00.162Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.159922792Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141652024-01-10T18:01:00.162ZWait for a query_work_queue command to finish before sending IO
141662024-01-10T18:01:00.162Z{"msg":"[2]R flush_numbers[0..12]: [219, 232, 232, 236, 234, 235, 235, 219, 237, 237, 237, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.159954448Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141672024-01-10T18:01:00.163Z{"msg":"[2]R generation[0..12]: [119, 128, 128, 130, 128, 129, 129, 119, 131, 131, 131, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.159995666Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141682024-01-10T18:01:00.163Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160020669Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141692024-01-10T18:01:00.163Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.1600423Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141702024-01-10T18:01:00.163Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160065048Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141712024-01-10T18:01:00.163Z{"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160109674Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141722024-01-10T18:01:00.163Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160132871Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"":"downstairs","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141732024-01-10T18:01:00.163Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.16015491Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141742024-01-10T18:01:00.163Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160207373Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"0","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141752024-01-10T18:01:00.165Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160230665Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"1","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141762024-01-10T18:01:00.165Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160252311Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"client":"2","":"downstairs","session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141772024-01-10T18:01:00.165Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160292797Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141782024-01-10T18:01:00.165Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160314789Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141792024-01-10T18:01:00.165Z{"msg":"1bebc3fc-6e70-4b26-8b3c-7364d3b4bca8 is now active with session: a303c634-565f-414d-9945-e6b4d493320a","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160336937Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141802024-01-10T18:01:00.165Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160374938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686,"session_id":"a303c634-565f-414d-9945-e6b4d493320a"}
141812024-01-10T18:01:00.165Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:00.160397022Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1686}
141822024-01-10T18:01:00.165ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
141832024-01-10T18:01:00.165ZLoading write count information from file "/tmp/test_repair_verify.data"
141842024-01-10T18:01:00.167ZRun Repair workload
141852024-01-10T18:01:00.167Z01/30 Write block 54 len 20480 data: 12 11 11 10 12
141862024-01-10T18:01:00.167Z02/30 Read block 550 len 28672
141872024-01-10T18:01:00.171Z03/30 Write block 137 len 40960 data: 9 10 11 12 13 11 10 10 9 9
141882024-01-10T18:01:00.171Z04/30 Read block 80 len 8192
141892024-01-10T18:01:00.171Z05/30 Flush
141902024-01-10T18:01:00.171Z06/30 Write block 71 len 8192 data: 6 5
141912024-01-10T18:01:00.171Z07/30 Write block 180 len 4096 data: 11
141922024-01-10T18:01:00.171Z08/30 Read block 395 len 28672
141932024-01-10T18:01:00.174Z09/30 Read block 1 len 4096
141942024-01-10T18:01:00.174Z10/30 Read block 572 len 32768
141952024-01-10T18:01:00.177Z11/30 Write block 335 len 8192 data: 8 9
141962024-01-10T18:01:00.177Z12/30 Write block 509 len 28672 data: 12 11 13 14 14 16 14
141972024-01-10T18:01:00.180Z13/30 Write block 360 len 12288 data: 6 6 8
141982024-01-10T18:01:00.180Z14/30 Flush
141992024-01-10T18:01:00.183Z15/30 Write block 276 len 8192 data: 12 11
142002024-01-10T18:01:00.183Z16/30 Read block 445 len 16384
142012024-01-10T18:01:00.186Z17/30 Read block 12 len 4096
142022024-01-10T18:01:00.186Z18/30 Write block 204 len 24576 data: 9 8 7 8 9 9
142032024-01-10T18:01:00.189Z19/30 Write block 36 len 8192 data: 13 11
142042024-01-10T18:01:00.189Z20/30 Write block 22 len 36864 data: 7 9 7 8 9 9 9 10 14
142052024-01-10T18:01:00.192Z21/30 Write block 515 len 16384 data: 15 13 12 11
142062024-01-10T18:01:00.192Z22/30 Write block 332 len 4096 data: 5
142072024-01-10T18:01:00.192Z23/30 Write block 121 len 16384 data: 10 10 8 6
142082024-01-10T18:01:00.195Z24/30 Write block 302 len 24576 data: 12 12 12 10 7 5
142092024-01-10T18:01:00.196Z25/30 Write block 466 len 36864 data: 11 12 12 8 8 5 4 5 6
142102024-01-10T18:01:00.196Z26/30 Read block 451 len 4096
142112024-01-10T18:01:00.199Z27/30 Read block 428 len 8192
142122024-01-10T18:01:00.199Z28/30 Write block 291 len 8192 data: 16 18
142132024-01-10T18:01:00.199Z29/30 Write block 209 len 12288 data: 10 9 10
142142024-01-10T18:01:00.205Z30/30 Read block 532 len 24576
142152024-01-10T18:01:00.205Z----------------------------------------------------------------
142162024-01-10T18:01:00.205Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
142172024-01-10T18:01:00.205ZStates: Active Active Active
142182024-01-10T18:01:00.205ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142192024-01-10T18:01:00.205Z 1 Acked 1000 Write 5 Sent Done Done false
142202024-01-10T18:01:00.205Z 2 Acked 1001 Read 7 Sent Done Done false
142212024-01-10T18:01:00.205Z 3 Acked 1002 Write 10 Sent Done Done false
142222024-01-10T18:01:00.205Z 4 Acked 1003 Read 2 Sent Done Done false
142232024-01-10T18:01:00.205Z 5 Acked 1004 Flush 0 Sent Done Done false
142242024-01-10T18:01:00.205Z 6 Acked 1005 Write 2 Sent Done Done false
142252024-01-10T18:01:00.206Z 7 Acked 1006 Write 1 Sent Done Done false
142262024-01-10T18:01:00.206Z 8 Acked 1007 Read 7 Sent Done Done false
142272024-01-10T18:01:00.206Z 9 Acked 1008 Read 1 Sent Done Done false
142282024-01-10T18:01:00.206Z 10 Acked 1009 Read 8 Sent Done Done false
142292024-01-10T18:01:00.206Z 11 Acked 1010 Write 2 Sent Done Done false
142302024-01-10T18:01:00.206Z 12 Acked 1011 Write 7 Sent Done Done false
142312024-01-10T18:01:00.206Z 13 Acked 1012 Write 3 Sent Done Done false
142322024-01-10T18:01:00.206Z 14 Acked 1013 Flush 0 Sent Done Done false
142332024-01-10T18:01:00.206Z 15 Acked 1014 Write 2 Sent Done Done false
142342024-01-10T18:01:00.206Z 16 Acked 1015 Read 4 Sent Done Done false
142352024-01-10T18:01:00.206Z 17 Acked 1016 Read 1 Sent Done Done false
142362024-01-10T18:01:00.209Z 18 Acked 1017 Write 6 Sent Done Done false
142372024-01-10T18:01:00.209Z 19 Acked 1018 Write 2 Sent Done Done false
142382024-01-10T18:01:00.210Z 20 Acked 1019 Write 9 Sent Done Done false
142392024-01-10T18:01:00.210Z 21 Acked 1020 Write 4 Sent Done Done false
142402024-01-10T18:01:00.210Z 22 Acked 1021 Write 1 Sent Done Done false
142412024-01-10T18:01:00.210Z 23 Acked 1022 Write 4 Sent Done Done false
142422024-01-10T18:01:00.210Z 24 Acked 1023 Write 6 Sent Done Done false
142432024-01-10T18:01:00.210Z 25 Acked 1024 Write 9 Sent Done Done false
142442024-01-10T18:01:00.210Z 26 Acked 1025 Read 1 Sent Done Done false
142452024-01-10T18:01:00.210Z 27 Acked 1026 Read 2 Sent Done Done false
142462024-01-10T18:01:00.210Z 28 Acked 1027 Write 2 Sent Done Done false
142472024-01-10T18:01:00.210Z 29 Acked 1028 Write 3 Sent Done Done false
142482024-01-10T18:01:00.210Z 30 Acked 1029 Read 6 Sent Done Sent false
142492024-01-10T18:01:00.210Z STATES DS:0 DS:1 DS:2 TOTAL
142502024-01-10T18:01:00.210Z New 0 0 0 0
142512024-01-10T18:01:00.210Z Sent 30 0 1 31
142522024-01-10T18:01:00.210Z Done 0 30 29 59
142532024-01-10T18:01:00.210Z Skipped 0 0 0 0
142542024-01-10T18:01:00.210Z Error 0 0 0 0
142552024-01-10T18:01:00.210ZLast Flush: 0 1013 1013
142562024-01-10T18:01:00.210ZDownstairs last five completed:
142572024-01-10T18:01:00.210ZUpstairs last five completed: 30 29 28 27 26
142582024-01-10T18:01:00.210ZWrote out file "/tmp/test_repair_verify.data"
142592024-01-10T18:01:00.214Z
142602024-01-10T18:01:00.216ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
142612024-01-10T18:01:00.243ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142622024-01-10T18:01:00.243Z 0 000-019 119 119 119 219 219 219 F F F
142632024-01-10T18:01:00.244Z 1 020-039 128 128 128 232 232 232 F T T <---
142642024-01-10T18:01:00.244Z 2 040-059 128 132 132 232 238 238 F F F <---
142652024-01-10T18:01:00.244Z 3 060-079 130 132 132 236 239 239 F F F <---
142662024-01-10T18:01:00.244Z 4 080-099 128 128 128 234 234 234 F F F
142672024-01-10T18:01:00.244Z 5 100-119 129 129 129 235 235 235 F F F
142682024-01-10T18:01:00.244Z 6 120-139 129 132 132 235 238 238 F T T <---
142692024-01-10T18:01:00.247Z 7 140-159 119 132 132 219 238 238 F F F <---
142702024-01-10T18:01:00.247Z 8 160-179 131 131 131 237 237 237 F F F
142712024-01-10T18:01:00.248Z 9 180-199 131 132 132 237 239 239 F F F <---
142722024-01-10T18:01:00.248Z 10 200-219 131 131 131 237 237 237 F T T <---
142732024-01-10T18:01:00.248Z 11 220-239 126 126 126 228 228 228 F F F
142742024-01-10T18:01:00.248Z 12 240-259 131 131 131 237 237 237 F F F
142752024-01-10T18:01:00.248Z 13 260-279 130 130 130 236 236 236 F T T <---
142762024-01-10T18:01:00.248Z 14 280-299 130 130 130 236 236 236 F T T <---
142772024-01-10T18:01:00.248Z 15 300-319 130 130 130 236 236 236 F T T <---
142782024-01-10T18:01:00.248Z 16 320-339 130 132 132 236 239 239 F T T <---
142792024-01-10T18:01:00.248Z 17 340-359 130 130 130 236 236 236 F F F
142802024-01-10T18:01:00.248Z 18 360-379 121 132 132 222 239 239 F F F <---
142812024-01-10T18:01:00.248Z 19 380-399 122 122 122 223 223 223 F F F
142822024-01-10T18:01:00.248Z 20 400-419 126 126 126 230 230 230 F F F
142832024-01-10T18:01:00.248Z 21 420-439 130 130 130 236 236 236 F F F
142842024-01-10T18:01:00.248Z 22 440-459 128 128 128 234 234 234 F F F
142852024-01-10T18:01:00.248Z 23 460-479 131 131 131 237 237 237 F T T <---
142862024-01-10T18:01:00.248Z 24 480-499 131 131 131 237 237 237 F F F
142872024-01-10T18:01:00.249Z 25 500-519 128 132 132 234 239 239 F T T <---
142882024-01-10T18:01:00.249Z 26 520-539 131 131 131 237 237 237 F F F
142892024-01-10T18:01:00.249Z 27 540-559 128 128 128 234 234 234 F F F
142902024-01-10T18:01:00.249Z 28 560-579 131 131 131 237 237 237 F F F
142912024-01-10T18:01:00.249Z 29 580-599 131 131 131 237 237 237 F F F
142922024-01-10T18:01:00.249ZMax gen: 132, Max flush: 239
142932024-01-10T18:01:00.249ZError: Difference in extent metadata found!
142942024-01-10T18:01:00.253ZA Difference in extent metadata is expected here
142952024-01-10T18:01:00.253ZOn loop 66
142962024-01-10T18:01:00.253Z
142972024-01-10T18:01:00.253ZVerifying data now
142982024-01-10T18:01:05.604ZNo change after verify
142992024-01-10T18:01:05.604ZLoop: 66 Downstairs dump after verify (and repair):
143002024-01-10T18:01:05.626ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143012024-01-10T18:01:05.626Z 0 000-019 119 119 119 219 219 219 F F F
143022024-01-10T18:01:05.626Z 1 020-039 133 133 133 240 240 240 F F F
143032024-01-10T18:01:05.626Z 2 040-059 132 132 132 238 238 238 F F F
143042024-01-10T18:01:05.626Z 3 060-079 132 132 132 239 239 239 F F F
143052024-01-10T18:01:05.630Z 4 080-099 128 128 128 234 234 234 F F F
143062024-01-10T18:01:05.630Z 5 100-119 129 129 129 235 235 235 F F F
143072024-01-10T18:01:05.630Z 6 120-139 133 133 133 240 240 240 F F F
143082024-01-10T18:01:05.630Z 7 140-159 132 132 132 238 238 238 F F F
143092024-01-10T18:01:05.630Z 8 160-179 131 131 131 237 237 237 F F F
143102024-01-10T18:01:05.630Z 9 180-199 132 132 132 239 239 239 F F F
143112024-01-10T18:01:05.630Z 10 200-219 133 133 133 240 240 240 F F F
143122024-01-10T18:01:05.630Z 11 220-239 126 126 126 228 228 228 F F F
143132024-01-10T18:01:05.630Z 12 240-259 131 131 131 237 237 237 F F F
143142024-01-10T18:01:05.630Z 13 260-279 133 133 133 240 240 240 F F F
143152024-01-10T18:01:05.630Z 14 280-299 133 133 133 240 240 240 F F F
143162024-01-10T18:01:05.630Z 15 300-319 133 133 133 240 240 240 F F F
143172024-01-10T18:01:05.630Z 16 320-339 133 133 133 240 240 240 F F F
143182024-01-10T18:01:05.630Z 17 340-359 130 130 130 236 236 236 F F F
143192024-01-10T18:01:05.630Z 18 360-379 132 132 132 239 239 239 F F F
143202024-01-10T18:01:05.630Z 19 380-399 122 122 122 223 223 223 F F F
143212024-01-10T18:01:05.630Z 20 400-419 126 126 126 230 230 230 F F F
143222024-01-10T18:01:05.630Z 21 420-439 130 130 130 236 236 236 F F F
143232024-01-10T18:01:05.630Z 22 440-459 128 128 128 234 234 234 F F F
143242024-01-10T18:01:05.630Z 23 460-479 133 133 133 240 240 240 F F F
143252024-01-10T18:01:05.630Z 24 480-499 131 131 131 237 237 237 F F F
143262024-01-10T18:01:05.631Z 25 500-519 133 133 133 240 240 240 F F F
143272024-01-10T18:01:05.631Z 26 520-539 131 131 131 237 237 237 F F F
143282024-01-10T18:01:05.631Z 27 540-559 128 128 128 234 234 234 F F F
143292024-01-10T18:01:05.631Z 28 560-579 131 131 131 237 237 237 F F F
143302024-01-10T18:01:05.631Z 29 580-599 131 131 131 237 237 237 F F F
143312024-01-10T18:01:05.631ZMax gen: 133, Max flush: 240
143322024-01-10T18:01:05.634Z
143332024-01-10T18:01:05.634ZBegin loop 67
143342024-01-10T18:01:05.634ZDownstairs to restart: 2
143352024-01-10T18:01:05.661Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.666733822Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694}
143362024-01-10T18:01:05.661Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.667260931Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694}
143372024-01-10T18:01:05.666Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.667333645Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694}
143382024-01-10T18:01:05.666Z{"msg":"Crucible 8021556e-12a7-469f-bd05-bd343ea0e284 has session id: dcb75dca-6c4d-41e7-bf81-f7e13ba3082a","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
143392024-01-10T18:01:05.666Z2024-01-10T18:01:05.667410675Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143402024-01-10T18:01:05.666Z{"msg":"Upstairs opts: Upstairs UUID: 8021556e-12a7-469f-bd05-bd343ea0e284, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.667677523Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143412024-01-10T18:01:05.666Z{"msg":"Crucible stats registered with UUID: 8021556e-12a7-469f-bd05-bd343ea0e284","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.667745383Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143422024-01-10T18:01:05.666Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.66777179Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694}
143432024-01-10T18:01:05.666Z{"msg":"8021556e-12a7-469f-bd05-bd343ea0e284 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.667936916Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143442024-01-10T18:01:05.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.667999226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"0","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143452024-01-10T18:01:05.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668087558Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"0","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143462024-01-10T18:01:05.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668150721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"1","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143472024-01-10T18:01:05.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668175742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"1","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143482024-01-10T18:01:05.667Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668203392Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"2","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143492024-01-10T18:01:05.667Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668243286Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"2","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143502024-01-10T18:01:05.667Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668308986Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143512024-01-10T18:01:05.667Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668337679Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143522024-01-10T18:01:05.667Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668419718Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143532024-01-10T18:01:05.668Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668448266Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143542024-01-10T18:01:05.668Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668715628Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143552024-01-10T18:01:05.668Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.668815972Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143562024-01-10T18:01:05.668Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.669529136Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"1","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143572024-01-10T18:01:05.668Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.669597774Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"1","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143582024-01-10T18:01:05.668Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.669623532Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"0","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143592024-01-10T18:01:05.668Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.669734688Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"0","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143602024-01-10T18:01:05.668Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.670274246Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"2","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143612024-01-10T18:01:05.668Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.670333117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"2","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143622024-01-10T18:01:05.668Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.670360085Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"0","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143632024-01-10T18:01:05.674Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.670863319Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"1","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143642024-01-10T18:01:05.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.671571466Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"0","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143652024-01-10T18:01:05.674Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.671639067Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"0","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143662024-01-10T18:01:05.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.67168273Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143672024-01-10T18:01:05.674Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.671740361Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"2","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143682024-01-10T18:01:05.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.671783956Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"1","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143692024-01-10T18:01:05.674Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.671836895Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"1","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143702024-01-10T18:01:05.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.671925609Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143712024-01-10T18:01:05.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.672637478Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"2","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143722024-01-10T18:01:05.674Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.672701541Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"2","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143732024-01-10T18:01:05.675Z{"msg":"[0]R flush_numbers[0..12]: [219, 240, 238, 239, 234, 235, 240, 238, 237, 239, 240, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.672729067Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143742024-01-10T18:01:05.676Z{"msg":"[0]R generation[0..12]: [119, 133, 132, 132, 128, 129, 133, 132, 131, 132, 133, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.672761783Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143752024-01-10T18:01:05.676Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.672818732Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143762024-01-10T18:01:05.676Z{"msg":"[1]R flush_numbers[0..12]: [219, 240, 238, 239, 234, 235, 240, 238, 237, 239, 240, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.672845565Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143772024-01-10T18:01:05.676Z{"msg":"[1]R generation[0..12]: [119, 133, 132, 132, 128, 129, 133, 132, 131, 132, 133, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.672902461Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143782024-01-10T18:01:05.676Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.672930959Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"Wait for a query_work_queue command to finish before sending IO
143792024-01-10T18:01:05.676Z}
143802024-01-10T18:01:05.676Z{"msg":"[2]R flush_numbers[0..12]: [219, 240, 238, 239, 234, 235, 240, 238, 237, 239, 240, 228]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673020817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143812024-01-10T18:01:05.676Z{"msg":"[2]R generation[0..12]: [119, 133, 132, 132, 128, 129, 133, 132, 131, 132, 133, 126]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673083252Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143822024-01-10T18:01:05.676Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673115713Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143832024-01-10T18:01:05.677Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673188286Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143842024-01-10T18:01:05.677Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.67323245Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143852024-01-10T18:01:05.677Z{"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673274434Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143862024-01-10T18:01:05.677Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673334359Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"":"downstairs","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143872024-01-10T18:01:05.677Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673376693Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143882024-01-10T18:01:05.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673437822Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"0","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143892024-01-10T18:01:05.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673477296Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"1","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143902024-01-10T18:01:05.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673519255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"client":"2","":"downstairs","session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143912024-01-10T18:01:05.678Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.67357776Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143922024-01-10T18:01:05.678Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673617417Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143932024-01-10T18:01:05.679Z{"msg":"8021556e-12a7-469f-bd05-bd343ea0e284 is now active with session: dcb75dca-6c4d-41e7-bf81-f7e13ba3082a","v":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
143942024-01-10T18:01:05.679Z0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673679807Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143952024-01-10T18:01:05.679Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.67371928Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694,"session_id":"dcb75dca-6c4d-41e7-bf81-f7e13ba3082a"}
143962024-01-10T18:01:05.679Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:05.673769389Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1694}
143972024-01-10T18:01:05.680ZLoading write count information from file "/tmp/test_repair_verify.data"
143982024-01-10T18:01:05.680ZRun Repair workload
143992024-01-10T18:01:05.680Z01/30 Write block 96 len 20480 data: 13 14 10 12 11
144002024-01-10T18:01:05.680Z02/30 Write block 352 len 8192 data: 9 9
144012024-01-10T18:01:05.680Z03/30 Write block 580 len 8192 data: 12 11
144022024-01-10T18:01:05.684Z04/30 Write block 145 len 16384 data: 10 10 11 11
144032024-01-10T18:01:05.684Z05/30 Write block 51 len 8192 data: 11 11
144042024-01-10T18:01:05.684Z06/30 Read block 144 len 4096
144052024-01-10T18:01:05.684Z07/30 Write block 152 len 24576 data: 8 9 10 11 12 10
144062024-01-10T18:01:05.684Z08/30 Read block 333 len 4096
144072024-01-10T18:01:05.685Z09/30 Write block 234 len 8192 data: 6 6
144082024-01-10T18:01:05.685Z10/30 Write block 343 len 40960 data: 14 13 13 13 14 9 8 7 8 10
144092024-01-10T18:01:05.685Z11/30 Read block 54 len 40960
144102024-01-10T18:01:05.688Z12/30 Read block 273 len 16384
144112024-01-10T18:01:05.688Z13/30 Read block 93 len 20480
144122024-01-10T18:01:05.688Z14/30 Flush
144132024-01-10T18:01:05.692Z15/30 Write block 489 len 8192 data: 10 12
144142024-01-10T18:01:05.695Z16/30 Read block 307 len 16384
144152024-01-10T18:01:05.695Z17/30 Read block 174 len 32768
144162024-01-10T18:01:05.705Z18/30 Read block 181 len 20480
144172024-01-10T18:01:05.705Z19/30 Flush
144182024-01-10T18:01:05.705Z20/30 Write block 384 len 12288 data: 6 7 7
144192024-01-10T18:01:05.705Z21/30 Write block 251 len 24576 data: 7 8 8 6 6 8
144202024-01-10T18:01:05.705Z22/30 Read block 508 len 20480
144212024-01-10T18:01:05.711Z23/30 Write block 239 len 32768 data: 9 9 9 11 13 17 16 11
144222024-01-10T18:01:05.711Z24/30 Read block 245 len 24576
144232024-01-10T18:01:05.711Z25/30 Write block 422 len 24576 data: 14 13 12 10 9 7
144242024-01-10T18:01:05.711Z26/30 Read block 252 len 32768
144252024-01-10T18:01:05.714Z27/30 Read block 154 len 24576
144262024-01-10T18:01:05.752Z28/30 Write block 116 len 28672 data: 10 10 9 11 11 11 11
144272024-01-10T18:01:05.752Z29/30 Write block 214 len 40960 data: 11 10 13 12 10 9 9 10 9 7
144282024-01-10T18:01:05.757Z30/30 Write block 337 len 12288 data: 9 9 11
144292024-01-10T18:01:05.757Z----------------------------------------------------------------
144302024-01-10T18:01:05.757Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
144312024-01-10T18:01:05.757ZStates: Active Active Active
144322024-01-10T18:01:05.757ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144332024-01-10T18:01:05.757Z 1 Acked 1000 Write 5 Done Done Done false
144342024-01-10T18:01:05.757Z 2 Acked 1001 Write 2 Done Done Done false
144352024-01-10T18:01:05.757Z 3 Acked 1002 Write 2 Done Done Done false
144362024-01-10T18:01:05.757Z 4 Acked 1003 Write 4 Done Done Done false
144372024-01-10T18:01:05.757Z 5 Acked 1004 Write 2 Done Done Sent false
144382024-01-10T18:01:05.758Z 6 Acked 1005 Read 1 Done Done Sent false
144392024-01-10T18:01:05.758Z 7 Acked 1006 Write 6 Done Done Sent false
144402024-01-10T18:01:05.758Z 8 Acked 1007 Read 1 Done Done Sent false
144412024-01-10T18:01:05.758Z 9 Acked 1008 Write 2 Done Done Sent false
144422024-01-10T18:01:05.758Z 10 Acked 1009 Write 10 Done Done Sent false
144432024-01-10T18:01:05.758Z 11 Acked 1010 Read 10 Done Done Sent false
144442024-01-10T18:01:05.758Z 12 Acked 1011 Read 4 Done Done Sent false
144452024-01-10T18:01:05.758Z 13 Acked 1012 Read 5 Done Done Sent false
144462024-01-10T18:01:05.758Z 14 Acked 1013 Flush 0 Done Done Sent false
144472024-01-10T18:01:05.759Z 15 Acked 1014 Write 2 Done Done Sent false
144482024-01-10T18:01:05.759Z 16 Acked 1015 Read 4 Done Done Sent false
144492024-01-10T18:01:05.759Z 17 Acked 1016 Read 8 Done Done Sent false
144502024-01-10T18:01:05.759Z 18 Acked 1017 Read 5 Done Done Sent false
144512024-01-10T18:01:05.759Z 19 Acked 1018 Flush 0 Done Done Sent false
144522024-01-10T18:01:05.759Z 20 Acked 1019 Write 3 Done Done Sent false
144532024-01-10T18:01:05.759Z 21 Acked 1020 Write 6 Done Done Sent false
144542024-01-10T18:01:05.759Z 22 Acked 1021 Read 5 Done Done Sent false
144552024-01-10T18:01:05.759Z 23 Acked 1022 Write 8 Done Done Sent false
144562024-01-10T18:01:05.759Z 24 Acked 1023 Read 6 Done Done Sent false
144572024-01-10T18:01:05.759Z 25 Acked 1024 Write 6 Done Done Sent false
144582024-01-10T18:01:05.759Z 26 Acked 1025 Read 8 Done Done Sent false
144592024-01-10T18:01:05.759Z 27 Acked 1026 Read 6 Done Done Sent false
144602024-01-10T18:01:05.759Z 28 Acked 1027 Write 7 Done Done Sent false
144612024-01-10T18:01:05.760Z 29 Acked 1028 Write 10 Sent Sent Sent false
144622024-01-10T18:01:05.760Z 30 Acked 1029 Write 3 Sent Sent Sent false
144632024-01-10T18:01:05.760Z STATES DS:0 DS:1 DS:2 TOTAL
144642024-01-10T18:01:05.760Z New 0 0 0 0
144652024-01-10T18:01:05.760Z Sent 2 2 26 30
144662024-01-10T18:01:05.760Z Done 28 28 4 60
144672024-01-10T18:01:05.760Z Skipped 0 0 0 0
144682024-01-10T18:01:05.760Z Error 0 0 0 0
144692024-01-10T18:01:05.760ZLast Flush: 1018 1018 0
144702024-01-10T18:01:05.760ZDownstairs last five completed:
144712024-01-10T18:01:05.760ZUpstairs last five completed: 30 29 28 27 26
144722024-01-10T18:01:05.761ZWrote out file "/tmp/test_repair_verify.data"
144732024-01-10T18:01:05.761Z
144742024-01-10T18:01:05.761ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
144752024-01-10T18:01:05.765ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144762024-01-10T18:01:05.765Z 0 000-019 119 119 119 219 219 219 F F F
144772024-01-10T18:01:05.765Z 1 020-039 133 133 133 240 240 240 F F F
144782024-01-10T18:01:05.765Z 2 040-059 134 134 132 241 241 238 F F F <---
144792024-01-10T18:01:05.765Z 3 060-079 132 132 132 239 239 239 F F F
144802024-01-10T18:01:05.765Z 4 080-099 134 134 128 241 241 234 F F T <---
144812024-01-10T18:01:05.765Z 5 100-119 134 134 129 241 241 235 T T T <---
144822024-01-10T18:01:05.765Z 6 120-139 133 133 133 240 240 240 T T F <---
144832024-01-10T18:01:05.765Z 7 140-159 134 134 132 241 241 238 F F T <---
144842024-01-10T18:01:05.765Z 8 160-179 131 131 131 237 237 237 F F F
144852024-01-10T18:01:05.765Z 9 180-199 132 132 132 239 239 239 F F F
144862024-01-10T18:01:05.767Z 10 200-219 133 133 133 240 240 240 T T F <---
144872024-01-10T18:01:05.767Z 11 220-239 134 134 126 241 241 228 T T F <---
144882024-01-10T18:01:05.767Z 12 240-259 131 131 131 237 237 237 T T F <---
144892024-01-10T18:01:05.767Z 13 260-279 133 133 133 240 240 240 F F F
144902024-01-10T18:01:05.767Z 14 280-299 133 133 133 240 240 240 F F F
144912024-01-10T18:01:05.767Z 15 300-319 133 133 133 240 240 240 F F F
144922024-01-10T18:01:05.767Z 16 320-339 133 133 133 240 240 240 T T F <---
144932024-01-10T18:01:05.767Z 17 340-359 134 134 130 241 241 236 F F T <---
144942024-01-10T18:01:05.767Z 18 360-379 132 132 132 239 239 239 F F F
144952024-01-10T18:01:05.767Z 19 380-399 122 122 122 223 223 223 T T F <---
144962024-01-10T18:01:05.767Z 20 400-419 126 126 126 230 230 230 F F F
144972024-01-10T18:01:05.767Z 21 420-439 130 130 130 236 236 236 T T F <---
144982024-01-10T18:01:05.767Z 22 440-459 128 128 128 234 234 234 F F F
144992024-01-10T18:01:05.767Z 23 460-479 133 133 133 240 240 240 F F F
145002024-01-10T18:01:05.767Z 24 480-499 134 134 131 242 242 237 F F F <---
145012024-01-10T18:01:05.767Z 25 500-519 133 133 133 240 240 240 F F F
145022024-01-10T18:01:05.767Z 26 520-539 131 131 131 237 237 237 F F F
145032024-01-10T18:01:05.768Z 27 540-559 128 128 128 234 234 234 F F F
145042024-01-10T18:01:05.768Z 28 560-579 131 131 131 237 237 237 F F F
145052024-01-10T18:01:05.768Z 29 580-599 134 134 131 241 241 237 F F T <---
145062024-01-10T18:01:05.768ZMax gen: 134, Max flush: 242
145072024-01-10T18:01:05.768ZError: Difference in extent metadata found!
145082024-01-10T18:01:05.768ZA Difference in extent metadata is expected here
145092024-01-10T18:01:05.768ZOn loop 67
145102024-01-10T18:01:05.768Z
145112024-01-10T18:01:05.773ZVerifying data now
145122024-01-10T18:01:12.121ZNo change after verify
145132024-01-10T18:01:12.121ZLoop: 67 Downstairs dump after verify (and repair):
145142024-01-10T18:01:12.146ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145152024-01-10T18:01:12.146Z 0 000-019 119 119 119 219 219 219 F F F
145162024-01-10T18:01:12.146Z 1 020-039 133 133 133 240 240 240 F F F
145172024-01-10T18:01:12.146Z 2 040-059 134 134 134 241 241 241 F F F
145182024-01-10T18:01:12.146Z 3 060-079 132 132 132 239 239 239 F F F
145192024-01-10T18:01:12.146Z 4 080-099 134 134 134 241 241 241 F F F
145202024-01-10T18:01:12.150Z 5 100-119 135 135 135 243 243 243 F F F
145212024-01-10T18:01:12.150Z 6 120-139 135 135 135 243 243 243 F F F
145222024-01-10T18:01:12.150Z 7 140-159 134 134 134 241 241 241 F F F
145232024-01-10T18:01:12.150Z 8 160-179 131 131 131 237 237 237 F F F
145242024-01-10T18:01:12.150Z 9 180-199 132 132 132 239 239 239 F F F
145252024-01-10T18:01:12.150Z 10 200-219 135 135 135 243 243 243 F F F
145262024-01-10T18:01:12.150Z 11 220-239 135 135 135 243 243 243 F F F
145272024-01-10T18:01:12.150Z 12 240-259 135 135 135 243 243 243 F F F
145282024-01-10T18:01:12.151Z 13 260-279 133 133 133 240 240 240 F F F
145292024-01-10T18:01:12.151Z 14 280-299 133 133 133 240 240 240 F F F
145302024-01-10T18:01:12.151Z 15 300-319 133 133 133 240 240 240 F F F
145312024-01-10T18:01:12.151Z 16 320-339 135 135 135 243 243 243 F F F
145322024-01-10T18:01:12.151Z 17 340-359 134 134 134 241 241 241 F F F
145332024-01-10T18:01:12.151Z 18 360-379 132 132 132 239 239 239 F F F
145342024-01-10T18:01:12.151Z 19 380-399 135 135 135 243 243 243 F F F
145352024-01-10T18:01:12.151Z 20 400-419 126 126 126 230 230 230 F F F
145362024-01-10T18:01:12.151Z 21 420-439 135 135 135 243 243 243 F F F
145372024-01-10T18:01:12.151Z 22 440-459 128 128 128 234 234 234 F F F
145382024-01-10T18:01:12.152Z 23 460-479 133 133 133 240 240 240 F F F
145392024-01-10T18:01:12.152Z 24 480-499 134 134 134 242 242 242 F F F
145402024-01-10T18:01:12.152Z 25 500-519 133 133 133 240 240 240 F F F
145412024-01-10T18:01:12.152Z 26 520-539 131 131 131 237 237 237 F F F
145422024-01-10T18:01:12.152Z 27 540-559 128 128 128 234 234 234 F F F
145432024-01-10T18:01:12.152Z 28 560-579 131 131 131 237 237 237 F F F
145442024-01-10T18:01:12.152Z 29 580-599 134 134 134 241 241 241 F F F
145452024-01-10T18:01:12.152ZMax gen: 135, Max flush: 243
145462024-01-10T18:01:12.152Z
145472024-01-10T18:01:12.152ZBegin loop 68
145482024-01-10T18:01:12.152ZDownstairs to restart: 1
145492024-01-10T18:01:12.177Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.18365488Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702}
145502024-01-10T18:01:12.178Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184124261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702}
145512024-01-10T18:01:12.183Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184194927Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702}
145522024-01-10T18:01:12.183Z{"msg":"Crucible 686135f6-a0c7-4038-9c89-e89687c0f9c1 has session id: fec740d4-08f2-4a8b-8def-f83458b9d16f","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184290899Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702Crucible runtime is spawned
145532024-01-10T18:01:12.183Z,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145542024-01-10T18:01:12.183Z{"msg":"Upstairs opts: Upstairs UUID: 686135f6-a0c7-4038-9c89-e89687c0f9c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184456446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145552024-01-10T18:01:12.183Z{"msg":"Crucible stats registered with UUID: 686135f6-a0c7-4038-9c89-e89687c0f9c1","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184583872Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145562024-01-10T18:01:12.184Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184612056Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702}
145572024-01-10T18:01:12.184Z{"msg":"686135f6-a0c7-4038-9c89-e89687c0f9c1 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184638883Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145582024-01-10T18:01:12.184Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184679238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"0","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145592024-01-10T18:01:12.184Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184704091Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"0","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145602024-01-10T18:01:12.184Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184726402Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"1","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145612024-01-10T18:01:12.185Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184748885Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"1","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145622024-01-10T18:01:12.185Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184770673Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"2","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145632024-01-10T18:01:12.185Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184802877Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"2","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145642024-01-10T18:01:12.185Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184886991Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145652024-01-10T18:01:12.185Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184915322Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145662024-01-10T18:01:12.185Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.184957475Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145672024-01-10T18:01:12.185Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.185374409Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145682024-01-10T18:01:12.185Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.185551788Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145692024-01-10T18:01:12.185Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.185601206Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145702024-01-10T18:01:12.185Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.186774184Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"0","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145712024-01-10T18:01:12.186Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.18686954Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"0","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145722024-01-10T18:01:12.186Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.186913718Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"2","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145732024-01-10T18:01:12.186Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.186990012Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"2","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145742024-01-10T18:01:12.186Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.187769189Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"1","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145752024-01-10T18:01:12.186Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.187833857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"1","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145762024-01-10T18:01:12.186Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.187861949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"0","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145772024-01-10T18:01:12.186Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.188026741Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"2","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145782024-01-10T18:01:12.191Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.188818816Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"2","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145792024-01-10T18:01:12.191Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.188925401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"2","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145802024-01-10T18:01:12.191Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.188986049Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145812024-01-10T18:01:12.192Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.189011059Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"1","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145822024-01-10T18:01:12.192Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.189057323Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"0","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145832024-01-10T18:01:12.192Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.189080867Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"0","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145842024-01-10T18:01:12.192Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.189161721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145852024-01-10T18:01:12.192Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.189813205Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"1","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145862024-01-10T18:01:12.192Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.189874855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"1","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145872024-01-10T18:01:12.192Z{"msg":"[0]R flush_numbers[0..12]: [219, 240, 241, 239, 241, 243, 243, 241, 237, 239, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.1899018Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145882024-01-10T18:01:12.192Z{"msg":"[0]R generation[0..12]: [119, 133, 134, 132, 134, 135, 135, 134, 131, 132, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.189925658Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145892024-01-10T18:01:12.192Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.189949436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145902024-01-10T18:01:12.192Z{"msg":"[1]R flush_numbers[0..12]: [219, 240, 241, 239, 241, 243, 243, 241, 237, 239, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.189972915Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145912024-01-10T18:01:12.193Z{"msg":"[1]R generation[0..12]: [119, 133, 134, 132, 134, 135, 135, 134, 131, 132, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190014226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145922024-01-10T18:01:12.193Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190039122Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145932024-01-10T18:01:12.193Z{"msg":"[2]R flush_numbers[0..12]: [219, 240, 241, 239, 241, 243, 243, 241, 237, 239, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190062463Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145942024-01-10T18:01:12.193Z{"msg":"[2]R generation[0..12]: [119, 133, 134, 132, 134, 135, 135, 134, 131, 132, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190085299Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145952024-01-10T18:01:12.193Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190108622Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145962024-01-10T18:01:12.193Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190129582Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145972024-01-10T18:01:12.193Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190151053Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
145982024-01-10T18:01:12.193ZWait for a query_work_queue command to finish before sending IO
145992024-01-10T18:01:12.193Z{"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190218695Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
146002024-01-10T18:01:12.193Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190255356Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"":"downstairs","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
146012024-01-10T18:01:12.194Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190311348Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
146022024-01-10T18:01:12.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.19035037Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"0","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
146032024-01-10T18:01:12.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190418472Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"1","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
146042024-01-10T18:01:12.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190456856Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"client":"2","":"downstairs","session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
146052024-01-10T18:01:12.195Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190519896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
146062024-01-10T18:01:12.195Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190558439Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
146072024-01-10T18:01:12.195Z{"msg":"686135f6-a0c7-4038-9c89-e89687c0f9c1 is now active with session: fec740d4-08f2-4a8b-8def-f83458b9d16f","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190611901Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
146082024-01-10T18:01:12.195Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190642938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702,"session_id":"fec740d4-08f2-4a8b-8def-f83458b9d16f"}
146092024-01-10T18:01:12.195Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:12.190704182Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1702}
146102024-01-10T18:01:12.195ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146112024-01-10T18:01:12.195ZLoading write count information from file "/tmp/test_repair_verify.data"
146122024-01-10T18:01:12.195ZRun Repair workload
146132024-01-10T18:01:12.195Z01/30 Write block 335 len 40960 data: 9 10 10 10 12 10 10 14 15 14
146142024-01-10T18:01:12.195Z02/30 Read block 263 len 36864
146152024-01-10T18:01:12.199Z03/30 Write block 481 len 40960 data: 6 6 8 11 8 9 8 9 11 13
146162024-01-10T18:01:12.199Z04/30 Read block 238 len 28672
146172024-01-10T18:01:12.199Z05/30 Flush
146182024-01-10T18:01:12.202Z06/30 Write block 190 len 32768 data: 14 14 13 14 16 18 16 16
146192024-01-10T18:01:12.202Z07/30 Read block 426 len 24576
146202024-01-10T18:01:12.205Z08/30 Read block 299 len 16384
146212024-01-10T18:01:12.208Z09/30 Read block 137 len 24576
146222024-01-10T18:01:12.208Z10/30 Flush
146232024-01-10T18:01:12.211Z11/30 Flush
146242024-01-10T18:01:12.214Z12/30 Write block 447 len 40960 data: 13 11 11 11 9 11 15 12 11 12
146252024-01-10T18:01:12.214Z13/30 Write block 565 len 4096 data: 10
146262024-01-10T18:01:12.214Z14/30 Flush
146272024-01-10T18:01:12.218Z15/30 Write block 522 len 12288 data: 14 9 8
146282024-01-10T18:01:12.218Z16/30 Read block 161 len 40960
146292024-01-10T18:01:12.221Z17/30 Read block 162 len 8192
146302024-01-10T18:01:12.224Z18/30 Read block 87 len 36864
146312024-01-10T18:01:12.227Z19/30 Write block 8 len 16384 data: 6 8 9 10
146322024-01-10T18:01:12.227Z20/30 Read block 399 len 16384
146332024-01-10T18:01:12.230Z21/30 Write block 543 len 32768 data: 10 11 12 12 15 15 16 17
146342024-01-10T18:01:12.230Z22/30 Write block 309 len 24576 data: 4 4 7 4 5 6
146352024-01-10T18:01:12.230Z23/30 Read block 47 len 8192
146362024-01-10T18:01:12.233Z24/30 Write block 113 len 28672 data: 15 12 10 11 11 10 12
146372024-01-10T18:01:12.236Z25/30 Write block 548 len 28672 data: 16 17 18 16 18 15 14
146382024-01-10T18:01:12.236Z26/30 Read block 570 len 24576
146392024-01-10T18:01:12.240Z27/30 Read block 316 len 36864
146402024-01-10T18:01:12.241Z28/30 Write block 35 len 12288 data: 13 14 12
146412024-01-10T18:01:12.244Z29/30 Read block 587 len 32768
146422024-01-10T18:01:12.244Z30/30 Read block 497 len 20480
146432024-01-10T18:01:12.248Z----------------------------------------------------------------
146442024-01-10T18:01:12.248Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30
146452024-01-10T18:01:12.248ZStates: Active Active Active
146462024-01-10T18:01:12.248ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
146472024-01-10T18:01:12.248Z 1 Acked 1000 Write 10 Done Sent Done false
146482024-01-10T18:01:12.248Z 2 Acked 1001 Read 9 Done Sent Done false
146492024-01-10T18:01:12.248Z 3 Acked 1002 Write 10 Done Sent Done false
146502024-01-10T18:01:12.248Z 4 Acked 1003 Read 7 Done Sent Done false
146512024-01-10T18:01:12.248Z 5 Acked 1004 Flush 0 Done Sent Done false
146522024-01-10T18:01:12.248Z 6 Acked 1005 Write 8 Done Sent Done false
146532024-01-10T18:01:12.248Z 7 Acked 1006 Read 6 Done Sent Done false
146542024-01-10T18:01:12.248Z 8 Acked 1007 Read 4 Done Sent Done false
146552024-01-10T18:01:12.248Z 9 Acked 1008 Read 6 Done Sent Done false
146562024-01-10T18:01:12.248Z 10 Acked 1009 Flush 0 Done Sent Done false
146572024-01-10T18:01:12.248Z 11 Acked 1010 Flush 0 Done Sent Done false
146582024-01-10T18:01:12.248Z 12 Acked 1011 Write 10 Done Sent Done false
146592024-01-10T18:01:12.248Z 13 Acked 1012 Write 1 Done Sent Done false
146602024-01-10T18:01:12.248Z 14 Acked 1013 Flush 0 Done Sent Done false
146612024-01-10T18:01:12.248Z 15 Acked 1014 Write 3 Done Sent Done false
146622024-01-10T18:01:12.248Z 16 Acked 1015 Read 10 Done Sent Done false
146632024-01-10T18:01:12.249Z 17 Acked 1016 Read 2 Done Sent Done false
146642024-01-10T18:01:12.249Z 18 Acked 1017 Read 9 Done Sent Done false
146652024-01-10T18:01:12.249Z 19 Acked 1018 Write 4 Done Sent Done false
146662024-01-10T18:01:12.249Z 20 Acked 1019 Read 4 Done Sent Done false
146672024-01-10T18:01:12.249Z 21 Acked 1020 Write 8 Done Sent Done false
146682024-01-10T18:01:12.249Z 22 Acked 1021 Write 6 Done Sent Done false
146692024-01-10T18:01:12.249Z 23 Acked 1022 Read 2 Done Sent Done false
146702024-01-10T18:01:12.249Z 24 Acked 1023 Write 7 Done Sent Done false
146712024-01-10T18:01:12.249Z 25 Acked 1024 Write 7 Done Sent Done false
146722024-01-10T18:01:12.249Z 26 Acked 1025 Read 6 Done Sent Done false
146732024-01-10T18:01:12.249Z 27 Acked 1026 Read 9 Done Sent Done false
146742024-01-10T18:01:12.250Z 28 Acked 1027 Write 3 Done Sent Done false
146752024-01-10T18:01:12.250Z 29 Acked 1028 Read 8 Done Sent Done false
146762024-01-10T18:01:12.250Z 30 Acked 1029 Read 5 Done Sent Done false
146772024-01-10T18:01:12.250Z STATES DS:0 DS:1 DS:2 TOTAL
146782024-01-10T18:01:12.250Z New 0 0 0 0
146792024-01-10T18:01:12.250Z Sent 0 30 0 30
146802024-01-10T18:01:12.250Z Done 30 0 30 60
146812024-01-10T18:01:12.250Z Skipped 0 0 0 0
146822024-01-10T18:01:12.250Z Error 0 0 0 0
146832024-01-10T18:01:12.250ZLast Flush: 1013 0 1013
146842024-01-10T18:01:12.250ZDownstairs last five completed:
146852024-01-10T18:01:12.250ZUpstairs last five completed: 30 29 28 27 26
146862024-01-10T18:01:12.250ZWrote out file "/tmp/test_repair_verify.data"
146872024-01-10T18:01:12.254Z
146882024-01-10T18:01:12.260ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
146892024-01-10T18:01:12.284ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146902024-01-10T18:01:12.284Z 0 000-019 119 119 119 219 219 219 T F T <---
146912024-01-10T18:01:12.284Z 1 020-039 133 133 133 240 240 240 T F T <---
146922024-01-10T18:01:12.285Z 2 040-059 134 134 134 241 241 241 F F F
146932024-01-10T18:01:12.285Z 3 060-079 132 132 132 239 239 239 F F F
146942024-01-10T18:01:12.285Z 4 080-099 134 134 134 241 241 241 F F F
146952024-01-10T18:01:12.288Z 5 100-119 135 135 135 243 243 243 T F T <---
146962024-01-10T18:01:12.288Z 6 120-139 135 135 135 243 243 243 F F F
146972024-01-10T18:01:12.288Z 7 140-159 134 134 134 241 241 241 F F F
146982024-01-10T18:01:12.288Z 8 160-179 131 131 131 237 237 237 F F F
146992024-01-10T18:01:12.288Z 9 180-199 136 132 136 245 239 245 F F F <---
147002024-01-10T18:01:12.288Z 10 200-219 135 135 135 243 243 243 F F F
147012024-01-10T18:01:12.288Z 11 220-239 135 135 135 243 243 243 F F F
147022024-01-10T18:01:12.288Z 12 240-259 135 135 135 243 243 243 F F F
147032024-01-10T18:01:12.288Z 13 260-279 133 133 133 240 240 240 F F F
147042024-01-10T18:01:12.288Z 14 280-299 133 133 133 240 240 240 F F F
147052024-01-10T18:01:12.288Z 15 300-319 133 133 133 240 240 240 T F T <---
147062024-01-10T18:01:12.288Z 16 320-339 136 135 136 244 243 244 F F F <---
147072024-01-10T18:01:12.288Z 17 340-359 136 134 136 244 241 244 F F F <---
147082024-01-10T18:01:12.288Z 18 360-379 132 132 132 239 239 239 F F F
147092024-01-10T18:01:12.288Z 19 380-399 135 135 135 243 243 243 F F F
147102024-01-10T18:01:12.288Z 20 400-419 126 126 126 230 230 230 F F F
147112024-01-10T18:01:12.288Z 21 420-439 135 135 135 243 243 243 F F F
147122024-01-10T18:01:12.288Z 22 440-459 136 128 136 247 234 247 F F F <---
147132024-01-10T18:01:12.288Z 23 460-479 133 133 133 240 240 240 F F F
147142024-01-10T18:01:12.289Z 24 480-499 136 134 136 244 242 244 F F F <---
147152024-01-10T18:01:12.289Z 25 500-519 133 133 133 240 240 240 F F F
147162024-01-10T18:01:12.289Z 26 520-539 131 131 131 237 237 237 T F T <---
147172024-01-10T18:01:12.289Z 27 540-559 128 128 128 234 234 234 T F T <---
147182024-01-10T18:01:12.289Z 28 560-579 136 131 136 247 237 247 F F F <---
147192024-01-10T18:01:12.289Z 29 580-599 134 134 134 241 241 241 F F F
147202024-01-10T18:01:12.289ZMax gen: 136, Max flush: 247
147212024-01-10T18:01:12.289ZError: Difference in extent metadata found!
147222024-01-10T18:01:12.295ZA Difference in extent metadata is expected here
147232024-01-10T18:01:12.295ZOn loop 68
147242024-01-10T18:01:12.295Z
147252024-01-10T18:01:12.298ZVerifying data now
147262024-01-10T18:01:17.510ZNo change after verify
147272024-01-10T18:01:17.510ZLoop: 68 Downstairs dump after verify (and repair):
147282024-01-10T18:01:17.533ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147292024-01-10T18:01:17.533Z 0 000-019 137 137 137 248 248 248 F F F
147302024-01-10T18:01:17.533Z 1 020-039 137 137 137 248 248 248 F F F
147312024-01-10T18:01:17.533Z 2 040-059 134 134 134 241 241 241 F F F
147322024-01-10T18:01:17.533Z 3 060-079 132 132 132 239 239 239 F F F
147332024-01-10T18:01:17.533Z 4 080-099 134 134 134 241 241 241 F F F
147342024-01-10T18:01:17.533Z 5 100-119 137 137 137 248 248 248 F F F
147352024-01-10T18:01:17.533Z 6 120-139 135 135 135 243 243 243 F F F
147362024-01-10T18:01:17.534Z 7 140-159 134 134 134 241 241 241 F F F
147372024-01-10T18:01:17.534Z 8 160-179 131 131 131 237 237 237 F F F
147382024-01-10T18:01:17.537Z 9 180-199 136 136 136 245 245 245 F F F
147392024-01-10T18:01:17.537Z 10 200-219 135 135 135 243 243 243 F F F
147402024-01-10T18:01:17.537Z 11 220-239 135 135 135 243 243 243 F F F
147412024-01-10T18:01:17.537Z 12 240-259 135 135 135 243 243 243 F F F
147422024-01-10T18:01:17.537Z 13 260-279 133 133 133 240 240 240 F F F
147432024-01-10T18:01:17.537Z 14 280-299 133 133 133 240 240 240 F F F
147442024-01-10T18:01:17.540Z 15 300-319 137 137 137 248 248 248 F F F
147452024-01-10T18:01:17.540Z 16 320-339 136 136 136 244 244 244 F F F
147462024-01-10T18:01:17.540Z 17 340-359 136 136 136 244 244 244 F F F
147472024-01-10T18:01:17.541Z 18 360-379 132 132 132 239 239 239 F F F
147482024-01-10T18:01:17.541Z 19 380-399 135 135 135 243 243 243 F F F
147492024-01-10T18:01:17.541Z 20 400-419 126 126 126 230 230 230 F F F
147502024-01-10T18:01:17.541Z 21 420-439 135 135 135 243 243 243 F F F
147512024-01-10T18:01:17.541Z 22 440-459 136 136 136 247 247 247 F F F
147522024-01-10T18:01:17.541Z 23 460-479 133 133 133 240 240 240 F F F
147532024-01-10T18:01:17.541Z 24 480-499 136 136 136 244 244 244 F F F
147542024-01-10T18:01:17.541Z 25 500-519 133 133 133 240 240 240 F F F
147552024-01-10T18:01:17.541Z 26 520-539 137 137 137 248 248 248 F F F
147562024-01-10T18:01:17.542Z 27 540-559 137 137 137 248 248 248 F F F
147572024-01-10T18:01:17.542Z 28 560-579 136 136 136 247 247 247 F F F
147582024-01-10T18:01:17.542Z 29 580-599 134 134 134 241 241 241 F F F
147592024-01-10T18:01:17.542ZMax gen: 137, Max flush: 248
147602024-01-10T18:01:17.546Z
147612024-01-10T18:01:17.546ZBegin loop 69
147622024-01-10T18:01:17.546ZDownstairs to restart: 0
147632024-01-10T18:01:17.567Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.573437056Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710}
147642024-01-10T18:01:17.571Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.573967177Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710}
147652024-01-10T18:01:17.571Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574040645Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710}
147662024-01-10T18:01:17.571Z{"msg":"Crucible 713cfcc1-9651-4a96-b6fd-e8e59b552df2 has session id: aa097bbf-cb97-4502-b90b-e388a4a02a87","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574158152Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147672024-01-10T18:01:17.571Z{"msg":"Crucible runtime is spawned
147682024-01-10T18:01:17.571ZUpstairs opts: Upstairs UUID: 713cfcc1-9651-4a96-b6fd-e8e59b552df2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574335905Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147692024-01-10T18:01:17.571Z{"msg":"Crucible stats registered with UUID: 713cfcc1-9651-4a96-b6fd-e8e59b552df2","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574444877Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147702024-01-10T18:01:17.571Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574470436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710}
147712024-01-10T18:01:17.571Z{"msg":"713cfcc1-9651-4a96-b6fd-e8e59b552df2 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574579477Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147722024-01-10T18:01:17.572Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.57464021Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"0","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147732024-01-10T18:01:17.572Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574666358Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"0","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147742024-01-10T18:01:17.572Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574691575Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"1","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147752024-01-10T18:01:17.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.57476261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"1","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147762024-01-10T18:01:17.573Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574807161Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"2","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147772024-01-10T18:01:17.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574876894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"2","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147782024-01-10T18:01:17.573Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.574972085Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147792024-01-10T18:01:17.573Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.575131359Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147802024-01-10T18:01:17.573Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.575210325Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147812024-01-10T18:01:17.573Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.575289989Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147822024-01-10T18:01:17.573Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.575337387Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147832024-01-10T18:01:17.573Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.575505255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147842024-01-10T18:01:17.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.576530321Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"2","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147852024-01-10T18:01:17.574Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.576593205Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"2","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147862024-01-10T18:01:17.574Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.576789818Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"1","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147872024-01-10T18:01:17.574Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.576875737Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"1","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147882024-01-10T18:01:17.574Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.57692684Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"0","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147892024-01-10T18:01:17.574Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.576964812Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"0","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147902024-01-10T18:01:17.581Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.577595134Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"2","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147912024-01-10T18:01:17.581Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.577743201Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"1","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147922024-01-10T18:01:17.581Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.578135613Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"0","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147932024-01-10T18:01:17.582Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.578650147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"2","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147942024-01-10T18:01:17.582Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.578783362Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"2","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147952024-01-10T18:01:17.582Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.578824327Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147962024-01-10T18:01:17.582Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.57888099Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"1","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147972024-01-10T18:01:17.582Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.578908136Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"1","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147982024-01-10T18:01:17.582Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.578930542Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
147992024-01-10T18:01:17.582Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.578985365Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"0","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148002024-01-10T18:01:17.582Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579013953Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"0","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148012024-01-10T18:01:17.582Z{"msg":"[0]R flush_numbers[0..12]: [248, 248, 241, 239, 241, 248, 243, 241, 237, 245, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579037135Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148022024-01-10T18:01:17.582Z{"msg":"[0]R generation[0..12]: [137, 137, 134, 132, 134, 137, 135, 134, 131, 136, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579079972Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148032024-01-10T18:01:17.582Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579104992Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148042024-01-10T18:01:17.582Z{"msg":"[1]R flush_numbers[0..12]: [248, 248, 241, 239, 241, 248, 243, 241, 237, 245, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579128132Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148052024-01-10T18:01:17.582Z{"msg":"[1]R generation[0..12]: [137, 137, 134, 132, 134, 137, 135, 134, 131, 136, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579150736Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148062024-01-10T18:01:17.583Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
148072024-01-10T18:01:17.583Z2024-01-10T18:01:17.579192463Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148082024-01-10T18:01:17.583Z{"msg":"[2]R flush_numbers[0..12]: [248, 248, 241, 239, 241, 248, 243, 241, 237, 245, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.57922077Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148092024-01-10T18:01:17.583Z{"msg":"[2]R generation[0..12]: [137, 137, 134, 132, 134, 137, 135, 134, 131, 136, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579243848Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148102024-01-10T18:01:17.583Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579286066Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148112024-01-10T18:01:17.583Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579309001Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148122024-01-10T18:01:17.583Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579330793Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148132024-01-10T18:01:17.583Z{"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579373444Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148142024-01-10T18:01:17.583Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579410657Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"":"downstairs","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148152024-01-10T18:01:17.585Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579467829Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148162024-01-10T18:01:17.585Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579489558Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"0","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148172024-01-10T18:01:17.585Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579511116Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"1","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148182024-01-10T18:01:17.585Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579564277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"client":"2","":"downstairs","session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148192024-01-10T18:01:17.585Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.57959915Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148202024-01-10T18:01:17.585Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.57964131Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148212024-01-10T18:01:17.585Z{"msg":"713cfcc1-9651-4a96-b6fd-e8e59b552df2 is now active with session: aa097bbf-cb97-4502-b90b-e388a4a02a87","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579664259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148222024-01-10T18:01:17.585Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579683311Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710,"session_id":"aa097bbf-cb97-4502-b90b-e388a4a02a87"}
148232024-01-10T18:01:17.585Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:17.579721065Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1710}
148242024-01-10T18:01:17.585ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148252024-01-10T18:01:17.585ZLoading write count information from file "/tmp/test_repair_verify.data"
148262024-01-10T18:01:17.586ZRun Repair workload
148272024-01-10T18:01:17.587Z01/30 Write block 421 len 32768 data: 16 15 14 13 11 10 8 8
148282024-01-10T18:01:17.587Z02/30 Write block 431 len 24576 data: 6 6 8 8 8 9
148292024-01-10T18:01:17.587Z03/30 Read block 58 len 28672
148302024-01-10T18:01:17.602Z04/30 Read block 163 len 36864
148312024-01-10T18:01:17.603Z05/30 Write block 313 len 20480 data: 6 7 10 9 7
148322024-01-10T18:01:17.603Z06/30 Write block 430 len 24576 data: 7 7 7 9 9 9
148332024-01-10T18:01:17.603Z07/30 Read block 92 len 8192
148342024-01-10T18:01:17.603Z08/30 Read block 304 len 24576
148352024-01-10T18:01:17.603Z09/30 Read block 257 len 28672
148362024-01-10T18:01:17.607Z10/30 Write block 582 len 8192 data: 11 12
148372024-01-10T18:01:17.607Z11/30 Read block 83 len 40960
148382024-01-10T18:01:17.607Z12/30 Read block 56 len 32768
148392024-01-10T18:01:17.607Z13/30 Write block 293 len 24576 data: 14 12 13 12 16 18
148402024-01-10T18:01:17.607Z14/30 Write block 91 len 24576 data: 12 11 11 11 14 14
148412024-01-10T18:01:17.607Z15/30 Read block 419 len 36864
148422024-01-10T18:01:17.607Z16/30 Write block 339 len 4096 data: 13
148432024-01-10T18:01:17.611Z17/30 Read block 201 len 16384
148442024-01-10T18:01:17.611Z18/30 Read block 556 len 16384
148452024-01-10T18:01:17.615Z19/30 Write block 74 len 36864 data: 8 11 11 11 7 9 8 9 9
148462024-01-10T18:01:17.615Z20/30 Write block 85 len 40960 data: 9 8 8 6 8 9 13 12 12 12
148472024-01-10T18:01:17.615Z21/30 Read block 140 len 16384
148482024-01-10T18:01:17.618Z22/30 Read block 514 len 28672
148492024-01-10T18:01:17.622Z23/30 Flush
148502024-01-10T18:01:17.625Z24/30 Flush
148512024-01-10T18:01:17.625Z25/30 Write block 370 len 16384 data: 9 8 8 9
148522024-01-10T18:01:17.629Z26/30 Write block 298 len 24576 data: 19 15 13 12 13 13
148532024-01-10T18:01:17.629Z27/30 Read block 336 len 32768
148542024-01-10T18:01:17.633Z28/30 Read block 529 len 4096
148552024-01-10T18:01:17.633Z29/30 Read block 162 len 32768
148562024-01-10T18:01:17.638Z30/30 Write block 241 len 12288 data: 10 12 14
148572024-01-10T18:01:17.638Z----------------------------------------------------------------
148582024-01-10T18:01:17.638Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
148592024-01-10T18:01:17.638ZStates: Active Active Active
148602024-01-10T18:01:17.638ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
148612024-01-10T18:01:17.638Z 1 Acked 1000 Write 8 Done Done Done false
148622024-01-10T18:01:17.638Z 2 Acked 1001 Write 6 Done Done Done false
148632024-01-10T18:01:17.638Z 3 Acked 1002 Read 7 Done Done Done false
148642024-01-10T18:01:17.638Z 4 Acked 1003 Read 9 Done Done Done false
148652024-01-10T18:01:17.639Z 5 Acked 1004 Write 5 Done Done Done false
148662024-01-10T18:01:17.639Z 6 Acked 1005 Write 6 Sent Done Done false
148672024-01-10T18:01:17.639Z 7 Acked 1006 Read 2 Sent Done Done false
148682024-01-10T18:01:17.639Z 8 Acked 1007 Read 6 Sent Done Done false
148692024-01-10T18:01:17.639Z 9 Acked 1008 Read 7 Sent Done Done false
148702024-01-10T18:01:17.639Z 10 Acked 1009 Write 2 Sent Done Done false
148712024-01-10T18:01:17.639Z 11 Acked 1010 Read 10 Sent Done Done false
148722024-01-10T18:01:17.639Z 12 Acked 1011 Read 8 Sent Done Done false
148732024-01-10T18:01:17.639Z 13 Acked 1012 Write 6 Sent Done Done false
148742024-01-10T18:01:17.639Z 14 Acked 1013 Write 6 Sent Done Done false
148752024-01-10T18:01:17.639Z 15 Acked 1014 Read 9 Sent Done Done false
148762024-01-10T18:01:17.639Z 16 Acked 1015 Write 1 Sent Done Done false
148772024-01-10T18:01:17.641Z 17 Acked 1016 Read 4 Sent Done Done false
148782024-01-10T18:01:17.641Z 18 Acked 1017 Read 4 Sent Done Done false
148792024-01-10T18:01:17.641Z 19 Acked 1018 Write 9 Sent Done Done false
148802024-01-10T18:01:17.641Z 20 Acked 1019 Write 10 Sent Done Done false
148812024-01-10T18:01:17.641Z 21 Acked 1020 Read 4 Sent Done Done false
148822024-01-10T18:01:17.641Z 22 Acked 1021 Read 7 Sent Done Done false
148832024-01-10T18:01:17.642Z 23 Acked 1022 Flush 0 Sent Done Done false
148842024-01-10T18:01:17.642Z 24 Acked 1023 Flush 0 Sent Done Done false
148852024-01-10T18:01:17.642Z 25 Acked 1024 Write 4 Sent Done Done false
148862024-01-10T18:01:17.642Z 26 Acked 1025 Write 6 Sent Done Done false
148872024-01-10T18:01:17.642Z 27 Acked 1026 Read 8 Sent Done Done false
148882024-01-10T18:01:17.642Z 28 Acked 1027 Read 1 Sent Done Done false
148892024-01-10T18:01:17.642Z 29 Acked 1028 Read 8 Sent Done Done false
148902024-01-10T18:01:17.642Z 30 Acked 1029 Write 3 Sent Sent Sent false
148912024-01-10T18:01:17.642Z STATES DS:0 DS:1 DS:2 TOTAL
148922024-01-10T18:01:17.642Z New 0 0 0 0
148932024-01-10T18:01:17.642Z Sent 25 1 1 27
148942024-01-10T18:01:17.642Z Done 5 29 29 63
148952024-01-10T18:01:17.643Z Skipped 0 0 0 0
148962024-01-10T18:01:17.643Z Error 0 0 0 0
148972024-01-10T18:01:17.643ZLast Flush: 0 1023 1023
148982024-01-10T18:01:17.643ZDownstairs last five completed:
148992024-01-10T18:01:17.643ZUpstairs last five completed: 30 29 28 27 26
149002024-01-10T18:01:17.643ZWrote out file "/tmp/test_repair_verify.data"
149012024-01-10T18:01:17.647Z
149022024-01-10T18:01:17.650ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149032024-01-10T18:01:17.673ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149042024-01-10T18:01:17.674Z 0 000-019 137 137 137 248 248 248 F F F
149052024-01-10T18:01:17.674Z 1 020-039 137 137 137 248 248 248 F F F
149062024-01-10T18:01:17.674Z 2 040-059 134 134 134 241 241 241 F F F
149072024-01-10T18:01:17.678Z 3 060-079 132 138 138 239 249 249 F F F <---
149082024-01-10T18:01:17.678Z 4 080-099 134 138 138 241 249 249 F F F <---
149092024-01-10T18:01:17.678Z 5 100-119 137 137 137 248 248 248 F F F
149102024-01-10T18:01:17.678Z 6 120-139 135 135 135 243 243 243 F F F
149112024-01-10T18:01:17.678Z 7 140-159 134 134 134 241 241 241 F F F
149122024-01-10T18:01:17.678Z 8 160-179 131 131 131 237 237 237 F F F
149132024-01-10T18:01:17.678Z 9 180-199 136 136 136 245 245 245 F F F
149142024-01-10T18:01:17.678Z 10 200-219 135 135 135 243 243 243 F F F
149152024-01-10T18:01:17.678Z 11 220-239 135 135 135 243 243 243 F F F
149162024-01-10T18:01:17.678Z 12 240-259 135 135 135 243 243 243 F T T <---
149172024-01-10T18:01:17.678Z 13 260-279 133 133 133 240 240 240 F F F
149182024-01-10T18:01:17.678Z 14 280-299 133 138 138 240 249 249 F T T <---
149192024-01-10T18:01:17.679Z 15 300-319 137 138 138 248 249 249 T T T <---
149202024-01-10T18:01:17.679Z 16 320-339 136 138 138 244 249 249 F F F <---
149212024-01-10T18:01:17.679Z 17 340-359 136 136 136 244 244 244 F F F
149222024-01-10T18:01:17.679Z 18 360-379 132 132 132 239 239 239 F T T <---
149232024-01-10T18:01:17.679Z 19 380-399 135 135 135 243 243 243 F F F
149242024-01-10T18:01:17.679Z 20 400-419 126 126 126 230 230 230 F F F
149252024-01-10T18:01:17.680Z 21 420-439 135 138 138 243 249 249 T F F <---
149262024-01-10T18:01:17.680Z 22 440-459 136 136 136 247 247 247 F F F
149272024-01-10T18:01:17.680Z 23 460-479 133 133 133 240 240 240 F F F
149282024-01-10T18:01:17.680Z 24 480-499 136 136 136 244 244 244 F F F
149292024-01-10T18:01:17.680Z 25 500-519 133 133 133 240 240 240 F F F
149302024-01-10T18:01:17.680Z 26 520-539 137 137 137 248 248 248 F F F
149312024-01-10T18:01:17.680Z 27 540-559 137 137 137 248 248 248 F F F
149322024-01-10T18:01:17.680Z 28 560-579 136 136 136 247 247 247 F F F
149332024-01-10T18:01:17.681Z 29 580-599 134 138 138 241 249 249 F F F <---
149342024-01-10T18:01:17.681ZMax gen: 138, Max flush: 249
149352024-01-10T18:01:17.682ZError: Difference in extent metadata found!
149362024-01-10T18:01:17.685ZA Difference in extent metadata is expected here
149372024-01-10T18:01:17.685ZOn loop 69
149382024-01-10T18:01:17.685Z
149392024-01-10T18:01:17.685ZVerifying data now
149402024-01-10T18:01:22.652ZNo change after verify
149412024-01-10T18:01:22.652ZLoop: 69 Downstairs dump after verify (and repair):
149422024-01-10T18:01:22.674ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149432024-01-10T18:01:22.674Z 0 000-019 137 137 137 248 248 248 F F F
149442024-01-10T18:01:22.674Z 1 020-039 137 137 137 248 248 248 F F F
149452024-01-10T18:01:22.674Z 2 040-059 134 134 134 241 241 241 F F F
149462024-01-10T18:01:22.674Z 3 060-079 138 138 138 249 249 249 F F F
149472024-01-10T18:01:22.674Z 4 080-099 138 138 138 249 249 249 F F F
149482024-01-10T18:01:22.674Z 5 100-119 137 137 137 248 248 248 F F F
149492024-01-10T18:01:22.674Z 6 120-139 135 135 135 243 243 243 F F F
149502024-01-10T18:01:22.674Z 7 140-159 134 134 134 241 241 241 F F F
149512024-01-10T18:01:22.674Z 8 160-179 131 131 131 237 237 237 F F F
149522024-01-10T18:01:22.675Z 9 180-199 136 136 136 245 245 245 F F F
149532024-01-10T18:01:22.681Z 10 200-219 135 135 135 243 243 243 F F F
149542024-01-10T18:01:22.681Z 11 220-239 135 135 135 243 243 243 F F F
149552024-01-10T18:01:22.681Z 12 240-259 139 139 139 250 250 250 F F F
149562024-01-10T18:01:22.681Z 13 260-279 133 133 133 240 240 240 F F F
149572024-01-10T18:01:22.681Z 14 280-299 139 139 139 250 250 250 F F F
149582024-01-10T18:01:22.682Z 15 300-319 139 139 139 250 250 250 F F F
149592024-01-10T18:01:22.682Z 16 320-339 138 138 138 249 249 249 F F F
149602024-01-10T18:01:22.682Z 17 340-359 136 136 136 244 244 244 F F F
149612024-01-10T18:01:22.682Z 18 360-379 139 139 139 250 250 250 F F F
149622024-01-10T18:01:22.682Z 19 380-399 135 135 135 243 243 243 F F F
149632024-01-10T18:01:22.682Z 20 400-419 126 126 126 230 230 230 F F F
149642024-01-10T18:01:22.682Z 21 420-439 138 138 138 249 249 249 F F F
149652024-01-10T18:01:22.682Z 22 440-459 136 136 136 247 247 247 F F F
149662024-01-10T18:01:22.682Z 23 460-479 133 133 133 240 240 240 F F F
149672024-01-10T18:01:22.682Z 24 480-499 136 136 136 244 244 244 F F F
149682024-01-10T18:01:22.682Z 25 500-519 133 133 133 240 240 240 F F F
149692024-01-10T18:01:22.682Z 26 520-539 137 137 137 248 248 248 F F F
149702024-01-10T18:01:22.682Z 27 540-559 137 137 137 248 248 248 F F F
149712024-01-10T18:01:22.682Z 28 560-579 136 136 136 247 247 247 F F F
149722024-01-10T18:01:22.682Z 29 580-599 138 138 138 249 249 249 F F F
149732024-01-10T18:01:22.682ZMax gen: 139, Max flush: 250
149742024-01-10T18:01:22.686Z
149752024-01-10T18:01:22.686ZBegin loop 70
149762024-01-10T18:01:22.686ZDownstairs to restart: 0
149772024-01-10T18:01:22.719Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.713979769Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718}
149782024-01-10T18:01:22.733Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.71451384Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718}
149792024-01-10T18:01:22.733Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.714603662Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718}
149802024-01-10T18:01:22.733Z{Crucible runtime is spawned
149812024-01-10T18:01:22.733Z"msg":"Crucible 2aee1f21-41a7-4bff-908b-685f1be12a06 has session id: ad589ff9-4465-419a-a31f-93bac0040844","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.714801858Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149822024-01-10T18:01:22.733Z{"msg":"Upstairs opts: Upstairs UUID: 2aee1f21-41a7-4bff-908b-685f1be12a06, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.714910472Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149832024-01-10T18:01:22.733Z{"msg":"Crucible stats registered with UUID: 2aee1f21-41a7-4bff-908b-685f1be12a06","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.714956896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149842024-01-10T18:01:22.733Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715018892Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718}
149852024-01-10T18:01:22.733Z{"msg":"2aee1f21-41a7-4bff-908b-685f1be12a06 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715068525Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149862024-01-10T18:01:22.733Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715111326Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"0","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149872024-01-10T18:01:22.733Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.71513612Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"0","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149882024-01-10T18:01:22.733Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715158527Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"1","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149892024-01-10T18:01:22.733Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715209445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"1","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149902024-01-10T18:01:22.733Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715250945Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"2","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149912024-01-10T18:01:22.733Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715319416Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"2","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149922024-01-10T18:01:22.733Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715361735Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149932024-01-10T18:01:22.733Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715414536Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149942024-01-10T18:01:22.733Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715461606Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149952024-01-10T18:01:22.734Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715525857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149962024-01-10T18:01:22.734Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715793832Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149972024-01-10T18:01:22.734Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.715865978Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149982024-01-10T18:01:22.734Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.716749235Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"1","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
149992024-01-10T18:01:22.736Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.716805261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"1","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150002024-01-10T18:01:22.736Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.716845459Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"2","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150012024-01-10T18:01:22.736Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.716882974Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"2","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150022024-01-10T18:01:22.736Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.717687868Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"2","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150032024-01-10T18:01:22.736Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.717952367Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"1","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150042024-01-10T18:01:22.736Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.718066787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"0","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150052024-01-10T18:01:22.736Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.718195286Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"0","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150062024-01-10T18:01:22.736Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.718495575Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"2","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150072024-01-10T18:01:22.736Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.718612484Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"2","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150082024-01-10T18:01:22.736Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.718660136Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150092024-01-10T18:01:22.739Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.719023218Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"1","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150102024-01-10T18:01:22.739Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.719085349Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"1","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150112024-01-10T18:01:22.739Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.719111524Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150122024-01-10T18:01:22.739Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.719154009Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"0","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150132024-01-10T18:01:22.739Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.71981548Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"0","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150142024-01-10T18:01:22.739Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.719884546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"0","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150152024-01-10T18:01:22.739Z{"msg":"[0]R flush_numbers[0..12]: [248, 248, 241, 249, 249, 248, 243, 241, 237, 245, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.71993226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150162024-01-10T18:01:22.739Z{"msg":"[0]R generation[0..12]: [137, 137, 134, 138, 138, 137, 135, 134, 131, 136, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.719988519Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150172024-01-10T18:01:22.739Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720034707Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150182024-01-10T18:01:22.739Z{"msg":"[1]R flush_numbers[0..12]: [248, 248, 241, 249, 249, 248, 243, 241, 237, 245, 243, 243]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
150192024-01-10T18:01:22.739Z30,"time":"2024-01-10T18:01:22.720136596Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150202024-01-10T18:01:22.739Z{"msg":"[1]R generation[0..12]: [137, 137, 134, 138, 138, 137, 135, 134, 131, 136, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720204897Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150212024-01-10T18:01:22.739Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720274996Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150222024-01-10T18:01:22.739Z{"msg":"[2]R flush_numbers[0..12]: [248, 248, 241, 249, 249, 248, 243, 241, 237, 245, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720318991Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150232024-01-10T18:01:22.739Z{"msg":"[2]R generation[0..12]: [137, 137, 134, 138, 138, 137, 135, 134, 131, 136, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.7203805Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150242024-01-10T18:01:22.739Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720445136Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150252024-01-10T18:01:22.739Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720475247Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150262024-01-10T18:01:22.740Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720497906Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150272024-01-10T18:01:22.740Z{"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720538444Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150282024-01-10T18:01:22.740Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720561556Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"":"downstairs","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150292024-01-10T18:01:22.740Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720583277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150302024-01-10T18:01:22.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720603082Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"0","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150312024-01-10T18:01:22.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720624549Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"1","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150322024-01-10T18:01:22.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720664334Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"client":"2","":"downstairs","session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150332024-01-10T18:01:22.740Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720687666Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150342024-01-10T18:01:22.740Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720708131Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150352024-01-10T18:01:22.740Z{"msg":"2aee1f21-41a7-4bff-908b-685f1be12a06 is now active with session: ad589ff9-4465-419a-a31f-93bac0040844","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720747339Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150362024-01-10T18:01:22.740Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720768205Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718,"session_id":"ad589ff9-4465-419a-a31f-93bac0040844"}
150372024-01-10T18:01:22.740Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:22.720788506Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1718}
150382024-01-10T18:01:22.740ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150392024-01-10T18:01:22.740ZLoading write count information from file "/tmp/test_repair_verify.data"
150402024-01-10T18:01:22.740ZRun Repair workload
150412024-01-10T18:01:22.741Z01/30 Flush
150422024-01-10T18:01:22.741Z02/30 Write block 286 len 16384 data: 11 11 10 13
150432024-01-10T18:01:22.741Z03/30 Read block 572 len 40960
150442024-01-10T18:01:22.741Z04/30 Write block 183 len 36864 data: 9 8 10 10 13 15 15 15 15
150452024-01-10T18:01:22.745Z05/30 Read block 125 len 16384
150462024-01-10T18:01:22.745Z06/30 Write block 321 len 8192 data: 10 9
150472024-01-10T18:01:22.745Z07/30 Read block 388 len 12288
150482024-01-10T18:01:22.745Z08/30 Write block 96 len 32768 data: 15 15 11 13 12 8 8 7
150492024-01-10T18:01:22.745Z09/30 Write block 331 len 20480 data: 7 6 5 5 10
150502024-01-10T18:01:22.745Z10/30 Read block 521 len 8192
150512024-01-10T18:01:22.745Z11/30 Read block 516 len 40960
150522024-01-10T18:01:22.745Z12/30 Write block 156 len 28672 data: 13 11 8 7 8 12 13
150532024-01-10T18:01:22.745Z13/30 Write block 593 len 24576 data: 9 10 8 7 6 5
150542024-01-10T18:01:22.746Z14/30 Read block 84 len 36864
150552024-01-10T18:01:22.746Z15/30 Write block 536 len 24576 data: 13 11 11 11 14 14
150562024-01-10T18:01:22.746Z16/30 Read block 23 len 12288
150572024-01-10T18:01:22.748Z17/30 Read block 442 len 28672
150582024-01-10T18:01:22.748Z18/30 Write block 522 len 8192 data: 15 10
150592024-01-10T18:01:22.749Z19/30 Read block 521 len 12288
150602024-01-10T18:01:22.751Z20/30 Read block 110 len 32768
150612024-01-10T18:01:22.754Z21/30 Read block 103 len 12288
150622024-01-10T18:01:22.757Z22/30 Read block 258 len 8192
150632024-01-10T18:01:22.758Z23/30 Flush
150642024-01-10T18:01:22.760Z24/30 Write block 425 len 36864 data: 12 11 9 9 9 8 8 8 10
150652024-01-10T18:01:22.763Z25/30 Read block 564 len 8192
150662024-01-10T18:01:22.763Z26/30 Read block 202 len 24576
150672024-01-10T18:01:22.766Z27/30 Read block 189 len 20480
150682024-01-10T18:01:22.769Z28/30 Read block 79 len 24576
150692024-01-10T18:01:22.773Z29/30 Read block 455 len 4096
150702024-01-10T18:01:22.773Z30/30 Write block 15 len 28672 data: 10 9 9 10 9 8 9
150712024-01-10T18:01:22.773Z----------------------------------------------------------------
150722024-01-10T18:01:22.773Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:29
150732024-01-10T18:01:22.773ZStates: Active Active Active
150742024-01-10T18:01:22.773ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
150752024-01-10T18:01:22.773Z 2 Acked 1001 Write 4 Done Done Done false
150762024-01-10T18:01:22.773Z 3 Acked 1002 Read 10 Done Done Done false
150772024-01-10T18:01:22.773Z 4 Acked 1003 Write 9 Done Done Done false
150782024-01-10T18:01:22.773Z 5 Acked 1004 Read 4 Done Done Done false
150792024-01-10T18:01:22.773Z 6 Acked 1005 Write 2 Sent Done Done false
150802024-01-10T18:01:22.773Z 7 Acked 1006 Read 3 Sent Done Done false
150812024-01-10T18:01:22.773Z 8 Acked 1007 Write 8 Sent Done Done false
150822024-01-10T18:01:22.773Z 9 Acked 1008 Write 5 Sent Done Done false
150832024-01-10T18:01:22.773Z 10 Acked 1009 Read 2 Sent Done Done false
150842024-01-10T18:01:22.773Z 11 Acked 1010 Read 10 Sent Done Done false
150852024-01-10T18:01:22.773Z 12 Acked 1011 Write 7 Sent Done Done false
150862024-01-10T18:01:22.773Z 13 Acked 1012 Write 6 Sent Done Done false
150872024-01-10T18:01:22.773Z 14 Acked 1013 Read 9 Sent Done Done false
150882024-01-10T18:01:22.774Z 15 Acked 1014 Write 6 Sent Done Done false
150892024-01-10T18:01:22.774Z 16 Acked 1015 Read 3 Sent Done Done false
150902024-01-10T18:01:22.774Z 17 Acked 1016 Read 7 Sent Done Done false
150912024-01-10T18:01:22.774Z 18 Acked 1017 Write 2 Sent Done Done false
150922024-01-10T18:01:22.774Z 19 Acked 1018 Read 3 Sent Done Done false
150932024-01-10T18:01:22.774Z 20 Acked 1019 Read 8 Sent Done Done false
150942024-01-10T18:01:22.774Z 21 Acked 1020 Read 3 Sent Done Done false
150952024-01-10T18:01:22.774Z 22 Acked 1021 Read 2 Sent Done Done false
150962024-01-10T18:01:22.774Z 23 Acked 1022 Flush 0 Sent Done Done false
150972024-01-10T18:01:22.774Z 24 Acked 1023 Write 9 Sent Done Done false
150982024-01-10T18:01:22.774Z 25 Acked 1024 Read 2 Sent Done Done false
150992024-01-10T18:01:22.774Z 26 Acked 1025 Read 6 Sent Done Done false
151002024-01-10T18:01:22.774Z 27 Acked 1026 Read 5 Sent Done Done false
151012024-01-10T18:01:22.774Z 28 Acked 1027 Read 6 Sent Done Done false
151022024-01-10T18:01:22.774Z 29 Acked 1028 Read 1 Sent Done Done false
151032024-01-10T18:01:22.774Z 30 Acked 1029 Write 7 Sent Sent Sent false
151042024-01-10T18:01:22.774Z STATES DS:0 DS:1 DS:2 TOTAL
151052024-01-10T18:01:22.774Z New 0 0 0 0
151062024-01-10T18:01:22.774Z Sent 25 1 1 27
151072024-01-10T18:01:22.774Z Done 4 28 28 60
151082024-01-10T18:01:22.774Z Skipped 0 0 0 0
151092024-01-10T18:01:22.774Z Error 0 0 0 0
151102024-01-10T18:01:22.774ZLast Flush: 1000 1022 1022
151112024-01-10T18:01:22.774ZDownstairs last five completed: 1000
151122024-01-10T18:01:22.774ZUpstairs last five completed: 30 29 28 27 26
151132024-01-10T18:01:22.778ZWrote out file "/tmp/test_repair_verify.data"
151142024-01-10T18:01:22.781Z
151152024-01-10T18:01:22.786ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151162024-01-10T18:01:22.810ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151172024-01-10T18:01:22.810Z 0 000-019 137 137 137 248 248 248 F T T <---
151182024-01-10T18:01:22.810Z 1 020-039 137 137 137 248 248 248 F T T <---
151192024-01-10T18:01:22.811Z 2 040-059 134 134 134 241 241 241 F F F
151202024-01-10T18:01:22.811Z 3 060-079 138 138 138 249 249 249 F F F
151212024-01-10T18:01:22.811Z 4 080-099 138 140 140 249 252 252 F F F <---
151222024-01-10T18:01:22.811Z 5 100-119 137 140 140 248 252 252 F F F <---
151232024-01-10T18:01:22.814Z 6 120-139 135 135 135 243 243 243 F F F
151242024-01-10T18:01:22.814Z 7 140-159 134 140 140 241 252 252 F F F <---
151252024-01-10T18:01:22.814Z 8 160-179 131 140 140 237 252 252 F F F <---
151262024-01-10T18:01:22.814Z 9 180-199 136 140 140 245 252 252 T F F <---
151272024-01-10T18:01:22.815Z 10 200-219 135 135 135 243 243 243 F F F
151282024-01-10T18:01:22.815Z 11 220-239 135 135 135 243 243 243 F F F
151292024-01-10T18:01:22.815Z 12 240-259 139 139 139 250 250 250 F F F
151302024-01-10T18:01:22.815Z 13 260-279 133 133 133 240 240 240 F F F
151312024-01-10T18:01:22.815Z 14 280-299 139 140 140 250 252 252 T F F <---
151322024-01-10T18:01:22.815Z 15 300-319 139 139 139 250 250 250 F F F
151332024-01-10T18:01:22.815Z 16 320-339 138 140 140 249 252 252 F F F <---
151342024-01-10T18:01:22.815Z 17 340-359 136 136 136 244 244 244 F F F
151352024-01-10T18:01:22.815Z 18 360-379 139 139 139 250 250 250 F F F
151362024-01-10T18:01:22.815Z 19 380-399 135 135 135 243 243 243 F F F
151372024-01-10T18:01:22.815Z 20 400-419 126 126 126 230 230 230 F F F
151382024-01-10T18:01:22.815Z 21 420-439 138 138 138 249 249 249 F T T <---
151392024-01-10T18:01:22.815Z 22 440-459 136 136 136 247 247 247 F F F
151402024-01-10T18:01:22.816Z 23 460-479 133 133 133 240 240 240 F F F
151412024-01-10T18:01:22.816Z 24 480-499 136 136 136 244 244 244 F F F
151422024-01-10T18:01:22.816Z 25 500-519 133 133 133 240 240 240 F F F
151432024-01-10T18:01:22.816Z 26 520-539 137 140 140 248 252 252 F F F <---
151442024-01-10T18:01:22.816Z 27 540-559 137 140 140 248 252 252 F F F <---
151452024-01-10T18:01:22.816Z 28 560-579 136 136 136 247 247 247 F F F
151462024-01-10T18:01:22.816Z 29 580-599 138 140 140 249 252 252 F F F <---
151472024-01-10T18:01:22.816ZMax gen: 140, Max flush: 252
151482024-01-10T18:01:22.816ZError: Difference in extent metadata found!
151492024-01-10T18:01:22.819ZA Difference in extent metadata is expected here
151502024-01-10T18:01:22.819ZOn loop 70
151512024-01-10T18:01:22.819Z
151522024-01-10T18:01:22.819ZVerifying data now
151532024-01-10T18:01:28.103ZNo change after verify
151542024-01-10T18:01:28.104ZLoop: 70 Downstairs dump after verify (and repair):
151552024-01-10T18:01:28.129ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151562024-01-10T18:01:28.129Z 0 000-019 141 141 141 253 253 253 F F F
151572024-01-10T18:01:28.129Z 1 020-039 141 141 141 253 253 253 F F F
151582024-01-10T18:01:28.129Z 2 040-059 134 134 134 241 241 241 F F F
151592024-01-10T18:01:28.129Z 3 060-079 138 138 138 249 249 249 F F F
151602024-01-10T18:01:28.130Z 4 080-099 140 140 140 252 252 252 F F F
151612024-01-10T18:01:28.130Z 5 100-119 140 140 140 252 252 252 F F F
151622024-01-10T18:01:28.130Z 6 120-139 135 135 135 243 243 243 F F F
151632024-01-10T18:01:28.130Z 7 140-159 140 140 140 252 252 252 F F F
151642024-01-10T18:01:28.130Z 8 160-179 140 140 140 252 252 252 F F F
151652024-01-10T18:01:28.130Z 9 180-199 140 140 140 252 252 252 F F F
151662024-01-10T18:01:28.139Z 10 200-219 135 135 135 243 243 243 F F F
151672024-01-10T18:01:28.139Z 11 220-239 135 135 135 243 243 243 F F F
151682024-01-10T18:01:28.139Z 12 240-259 139 139 139 250 250 250 F F F
151692024-01-10T18:01:28.140Z 13 260-279 133 133 133 240 240 240 F F F
151702024-01-10T18:01:28.140Z 14 280-299 140 140 140 252 252 252 F F F
151712024-01-10T18:01:28.141Z 15 300-319 139 139 139 250 250 250 F F F
151722024-01-10T18:01:28.141Z 16 320-339 140 140 140 252 252 252 F F F
151732024-01-10T18:01:28.141Z 17 340-359 136 136 136 244 244 244 F F F
151742024-01-10T18:01:28.141Z 18 360-379 139 139 139 250 250 250 F F F
151752024-01-10T18:01:28.141Z 19 380-399 135 135 135 243 243 243 F F F
151762024-01-10T18:01:28.141Z 20 400-419 126 126 126 230 230 230 F F F
151772024-01-10T18:01:28.141Z 21 420-439 141 141 141 253 253 253 F F F
151782024-01-10T18:01:28.141Z 22 440-459 136 136 136 247 247 247 F F F
151792024-01-10T18:01:28.141Z 23 460-479 133 133 133 240 240 240 F F F
151802024-01-10T18:01:28.142Z 24 480-499 136 136 136 244 244 244 F F F
151812024-01-10T18:01:28.142Z 25 500-519 133 133 133 240 240 240 F F F
151822024-01-10T18:01:28.142Z 26 520-539 140 140 140 252 252 252 F F F
151832024-01-10T18:01:28.142Z 27 540-559 140 140 140 252 252 252 F F F
151842024-01-10T18:01:28.142Z 28 560-579 136 136 136 247 247 247 F F F
151852024-01-10T18:01:28.142Z 29 580-599 140 140 140 252 252 252 F F F
151862024-01-10T18:01:28.143ZMax gen: 141, Max flush: 253
151872024-01-10T18:01:28.143Z
151882024-01-10T18:01:28.143ZBegin loop 71
151892024-01-10T18:01:28.143ZDownstairs to restart: 0
151902024-01-10T18:01:28.157Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.163594706Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726}
151912024-01-10T18:01:28.158Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164106242Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726}
151922024-01-10T18:01:28.172Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164169546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726}
151932024-01-10T18:01:28.173Z{"msg":"Crucible 405d511c-255a-487d-98d1-5b119a413312 has session id: ef83b039-4dee-4bda-bf27-10518db7752a","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164228289Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
151942024-01-10T18:01:28.173Z{"msg":"Upstairs opts: Upstairs UUID: 405d511c-255a-487d-98d1-5b119a413312, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164289143Z","hostname":Crucible runtime is spawned
151952024-01-10T18:01:28.173Z"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
151962024-01-10T18:01:28.173Z{"msg":"Crucible stats registered with UUID: 405d511c-255a-487d-98d1-5b119a413312","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164368622Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
151972024-01-10T18:01:28.173Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164428149Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726}
151982024-01-10T18:01:28.174Z{"msg":"405d511c-255a-487d-98d1-5b119a413312 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164630813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
151992024-01-10T18:01:28.174Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164680431Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"0","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152002024-01-10T18:01:28.174Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164716709Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"0","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152012024-01-10T18:01:28.177Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.16483859Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"1","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152022024-01-10T18:01:28.177Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164908607Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"1","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152032024-01-10T18:01:28.177Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.164938102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"2","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152042024-01-10T18:01:28.177Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.165046595Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"2","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152052024-01-10T18:01:28.177Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.165298448Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152062024-01-10T18:01:28.178Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.165359717Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152072024-01-10T18:01:28.178Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.165386052Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152082024-01-10T18:01:28.178Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.165413128Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152092024-01-10T18:01:28.178Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.165490646Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152102024-01-10T18:01:28.179Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.165524602Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152112024-01-10T18:01:28.179Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.166497954Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"1","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152122024-01-10T18:01:28.180Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.166582652Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"1","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152132024-01-10T18:01:28.180Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.166629498Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"2","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152142024-01-10T18:01:28.181Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.166673109Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"2","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152152024-01-10T18:01:28.181Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.167414742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"0","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152162024-01-10T18:01:28.181Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.167457266Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"0","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152172024-01-10T18:01:28.181Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.167602731Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"2","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152182024-01-10T18:01:28.181Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.167693686Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"1","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152192024-01-10T18:01:28.181Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.168507147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"1","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152202024-01-10T18:01:28.182Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.168569743Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"1","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152212024-01-10T18:01:28.182Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.168595757Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152222024-01-10T18:01:28.182Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.168616489Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"2","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152232024-01-10T18:01:28.183Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.168638485Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"2","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152242024-01-10T18:01:28.183Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.168661038Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152252024-01-10T18:01:28.183Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.168690395Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"0","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152262024-01-10T18:01:28.183Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169373943Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"0","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152272024-01-10T18:01:28.183Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169449841Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"0","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152282024-01-10T18:01:28.183Z{"msg":"[0]R flush_numbers[0..12]: [253, 253, 241, 249, 252, 252, 243, 252, 252, 252, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169494958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152292024-01-10T18:01:28.183Z{"msg":"[0]R generation[0..12]: [141, 141, 134, 138, 140, 140, 135, 140, 140, 140, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169543183Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152302024-01-10T18:01:28.183Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.16956964Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152312024-01-10T18:01:28.184Z{"msg":"[1]R flush_numbers[0..12]: [253, 253, 241, 249, 252, 252, 243, 252, 252, 252, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169593122Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152322024-01-10T18:01:28.184Z{"msg":"[1]R generation[0..12]: [141, 141, 134, 138, 140, 140, 135, 140, 140, 140, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169635102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152332024-01-10T18:01:28.184Z{"msgWait for a query_work_queue command to finish before sending IO
152342024-01-10T18:01:28.184Z":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169663811Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152352024-01-10T18:01:28.184Z{"msg":"[2]R flush_numbers[0..12]: [253, 253, 241, 249, 252, 252, 243, 252, 252, 252, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169704992Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152362024-01-10T18:01:28.184Z{"msg":"[2]R generation[0..12]: [141, 141, 134, 138, 140, 140, 135, 140, 140, 140, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169729276Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152372024-01-10T18:01:28.184Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169771891Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152382024-01-10T18:01:28.184Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169794935Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152392024-01-10T18:01:28.184Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169818317Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152402024-01-10T18:01:28.185Z{"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.16983942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152412024-01-10T18:01:28.607Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169862102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"":"downstairs","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152422024-01-10T18:01:28.607Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169901417Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152432024-01-10T18:01:28.608Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169922711Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"0","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152442024-01-10T18:01:28.608Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169964788Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"1","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152452024-01-10T18:01:28.608Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.169987514Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"client":"2","":"downstairs","session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152462024-01-10T18:01:28.608Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.170009014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152472024-01-10T18:01:28.609Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.170028838Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152482024-01-10T18:01:28.609Z{"msg":"405d511c-255a-487d-98d1-5b119a413312 is now active with session: ef83b039-4dee-4bda-bf27-10518db7752a","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.170049866Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152492024-01-10T18:01:28.609Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.170068961Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726,"session_id":"ef83b039-4dee-4bda-bf27-10518db7752a"}
152502024-01-10T18:01:28.609Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:28.170114383Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1726}
152512024-01-10T18:01:28.609ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
152522024-01-10T18:01:28.609ZLoading write count information from file "/tmp/test_repair_verify.data"
152532024-01-10T18:01:28.610ZRun Repair workload
152542024-01-10T18:01:28.610Z01/30 Write block 313 len 16384 data: 7 8 11 10
152552024-01-10T18:01:28.610Z02/30 Read block 411 len 28672
152562024-01-10T18:01:28.610Z03/30 Write block 121 len 12288 data: 12 12 9
152572024-01-10T18:01:28.610Z04/30 Read block 434 len 40960
152582024-01-10T18:01:28.610Z05/30 Read block 318 len 28672
152592024-01-10T18:01:28.611Z06/30 Write block 356 len 12288 data: 8 9 11
152602024-01-10T18:01:28.611Z07/30 Flush
152612024-01-10T18:01:28.611Z08/30 Write block 375 len 4096 data: 8
152622024-01-10T18:01:28.611Z09/30 Read block 500 len 8192
152632024-01-10T18:01:28.611Z10/30 Write block 532 len 24576 data: 13 14 14 13 14 12
152642024-01-10T18:01:28.611Z11/30 Read block 456 len 24576
152652024-01-10T18:01:28.612Z12/30 Read block 97 len 20480
152662024-01-10T18:01:28.612Z13/30 Flush
152672024-01-10T18:01:28.629Z14/30 Write block 551 len 24576 data: 17 19 16 15 12 11
152682024-01-10T18:01:28.629Z15/30 Read block 513 len 20480
152692024-01-10T18:01:28.629Z16/30 Read block 501 len 36864
152702024-01-10T18:01:28.629Z17/30 Read block 547 len 16384
152712024-01-10T18:01:28.629Z18/30 Read block 237 len 24576
152722024-01-10T18:01:28.629Z19/30 Write block 510 len 12288 data: 12 14 15
152732024-01-10T18:01:28.629Z20/30 Read block 509 len 32768
152742024-01-10T18:01:28.629Z21/30 Write block 594 len 8192 data: 11 9
152752024-01-10T18:01:28.629Z22/30 Read block 27 len 20480
152762024-01-10T18:01:28.630Z23/30 Read block 401 len 24576
152772024-01-10T18:01:28.630Z24/30 Read block 73 len 24576
152782024-01-10T18:01:28.630Z25/30 Write block 574 len 16384 data: 11 12 13 11
152792024-01-10T18:01:28.630Z26/30 Read block 266 len 36864
152802024-01-10T18:01:28.631Z27/30 Read block 453 len 32768
152812024-01-10T18:01:28.631Z28/30 Write block 378 len 12288 data: 5 6 7
152822024-01-10T18:01:28.631Z29/30 Read block 499 len 20480
152832024-01-10T18:01:28.631Z30/30 Write block 446 len 36864 data: 14 14 12 12 12 10 12 16 13
152842024-01-10T18:01:28.631Z----------------------------------------------------------------
152852024-01-10T18:01:28.631Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
152862024-01-10T18:01:28.631ZStates: Active Active Active
152872024-01-10T18:01:28.631ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
152882024-01-10T18:01:28.631Z 1 Acked 1000 Write 4 Sent Done Done false
152892024-01-10T18:01:28.631Z 2 Acked 1001 Read 7 Sent Done Done false
152902024-01-10T18:01:28.631Z 3 Acked 1002 Write 3 Sent Done Done false
152912024-01-10T18:01:28.631Z 4 Acked 1003 Read 10 Sent Done Done false
152922024-01-10T18:01:28.632Z 5 Acked 1004 Read 7 Sent Done Done false
152932024-01-10T18:01:28.632Z 6 Acked 1005 Write 3 Sent Done Done false
152942024-01-10T18:01:28.632Z 7 Acked 1006 Flush 0 Sent Done Done false
152952024-01-10T18:01:28.632Z 8 Acked 1007 Write 1 Sent Done Done false
152962024-01-10T18:01:28.632Z 9 Acked 1008 Read 2 Sent Done Done false
152972024-01-10T18:01:28.632Z 10 Acked 1009 Write 6 Sent Done Done false
152982024-01-10T18:01:28.632Z 11 Acked 1010 Read 6 Sent Done Done false
152992024-01-10T18:01:28.632Z 12 Acked 1011 Read 5 Sent Done Done false
153002024-01-10T18:01:28.632Z 13 Acked 1012 Flush 0 Sent Done Done false
153012024-01-10T18:01:28.632Z 14 Acked 1013 Write 6 Sent Done Done false
153022024-01-10T18:01:28.632Z 15 Acked 1014 Read 5 Sent Done Done false
153032024-01-10T18:01:28.634Z 16 Acked 1015 Read 9 Sent Done Done false
153042024-01-10T18:01:28.634Z 17 Acked 1016 Read 4 Sent Done Done false
153052024-01-10T18:01:28.636Z 18 Acked 1017 Read 6 Sent Done Done false
153062024-01-10T18:01:28.636Z 19 Acked 1018 Write 3 Sent Done Done false
153072024-01-10T18:01:28.636Z 20 Acked 1019 Read 8 Sent Done Done false
153082024-01-10T18:01:28.636Z 21 Acked 1020 Write 2 Sent Done Done false
153092024-01-10T18:01:28.636Z 22 Acked 1021 Read 5 Sent Done Done false
153102024-01-10T18:01:28.636Z 23 Acked 1022 Read 6 Sent Done Done false
153112024-01-10T18:01:28.636Z 24 Acked 1023 Read 6 Sent Done Done false
153122024-01-10T18:01:28.637Z 25 Acked 1024 Write 4 Sent Done Done false
153132024-01-10T18:01:28.637Z 26 Acked 1025 Read 9 Sent Done Done false
153142024-01-10T18:01:28.637Z 27 Acked 1026 Read 8 Sent Done Done false
153152024-01-10T18:01:28.637Z 28 Acked 1027 Write 3 Sent Done Done false
153162024-01-10T18:01:28.637Z 29 Acked 1028 Read 5 Sent Done Done false
153172024-01-10T18:01:28.637Z 30 Acked 1029 Write 9 Sent Sent Sent false
153182024-01-10T18:01:28.637Z STATES DS:0 DS:1 DS:2 TOTAL
153192024-01-10T18:01:28.637Z New 0 0 0 0
153202024-01-10T18:01:28.637Z Sent 30 1 1 32
153212024-01-10T18:01:28.637Z Done 0 29 29 58
153222024-01-10T18:01:28.637Z Skipped 0 0 0 0
153232024-01-10T18:01:28.637Z Error 0 0 0 0
153242024-01-10T18:01:28.637ZLast Flush: 0 1012 1012
153252024-01-10T18:01:28.638ZDownstairs last five completed:
153262024-01-10T18:01:28.638ZUpstairs last five completed: 30 29 28 27 26
153272024-01-10T18:01:28.639ZWrote out file "/tmp/test_repair_verify.data"
153282024-01-10T18:01:28.639Z
153292024-01-10T18:01:28.639ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153302024-01-10T18:01:28.639ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153312024-01-10T18:01:28.639Z 0 000-019 141 141 141 253 253 253 F F F
153322024-01-10T18:01:28.639Z 1 020-039 141 141 141 253 253 253 F F F
153332024-01-10T18:01:28.639Z 2 040-059 134 134 134 241 241 241 F F F
153342024-01-10T18:01:28.639Z 3 060-079 138 138 138 249 249 249 F F F
153352024-01-10T18:01:28.639Z 4 080-099 140 140 140 252 252 252 F F F
153362024-01-10T18:01:28.639Z 5 100-119 140 140 140 252 252 252 F F F
153372024-01-10T18:01:28.639Z 6 120-139 135 142 142 243 254 254 F F F <---
153382024-01-10T18:01:28.639Z 7 140-159 140 140 140 252 252 252 F F F
153392024-01-10T18:01:28.640Z 8 160-179 140 140 140 252 252 252 F F F
153402024-01-10T18:01:28.640Z 9 180-199 140 140 140 252 252 252 F F F
153412024-01-10T18:01:28.640Z 10 200-219 135 135 135 243 243 243 F F F
153422024-01-10T18:01:28.640Z 11 220-239 135 135 135 243 243 243 F F F
153432024-01-10T18:01:28.640Z 12 240-259 139 139 139 250 250 250 F F F
153442024-01-10T18:01:28.640Z 13 260-279 133 133 133 240 240 240 F F F
153452024-01-10T18:01:28.640Z 14 280-299 140 140 140 252 252 252 F F F
153462024-01-10T18:01:28.640Z 15 300-319 139 142 142 250 254 254 F F F <---
153472024-01-10T18:01:28.640Z 16 320-339 140 140 140 252 252 252 F F F
153482024-01-10T18:01:28.640Z 17 340-359 136 142 142 244 254 254 F F F <---
153492024-01-10T18:01:28.640Z 18 360-379 139 142 142 250 255 255 F T T <---
153502024-01-10T18:01:28.640Z 19 380-399 135 135 135 243 243 243 F T T <---
153512024-01-10T18:01:28.641Z 20 400-419 126 126 126 230 230 230 F F F
153522024-01-10T18:01:28.641Z 21 420-439 141 141 141 253 253 253 F F F
153532024-01-10T18:01:28.641Z 22 440-459 136 136 136 247 247 247 F T T <---
153542024-01-10T18:01:28.641Z 23 460-479 133 133 133 240 240 240 F F F
153552024-01-10T18:01:28.641Z 24 480-499 136 136 136 244 244 244 F F F
153562024-01-10T18:01:28.642Z 25 500-519 133 133 133 240 240 240 F T T <---
153572024-01-10T18:01:28.642Z 26 520-539 140 142 142 252 255 255 F F F <---
153582024-01-10T18:01:28.642Z 27 540-559 140 140 140 252 252 252 F T T <---
153592024-01-10T18:01:28.642Z 28 560-579 136 136 136 247 247 247 F T T <---
153602024-01-10T18:01:28.642Z 29 580-599 140 140 140 252 252 252 F T T <---
153612024-01-10T18:01:28.642ZMax gen: 142, Max flush: 255
153622024-01-10T18:01:28.642ZError: Difference in extent metadata found!
153632024-01-10T18:01:28.642ZA Difference in extent metadata is expected here
153642024-01-10T18:01:28.642ZOn loop 71
153652024-01-10T18:01:28.642Z
153662024-01-10T18:01:28.643ZVerifying data now
153672024-01-10T18:01:33.402ZNo change after verify
153682024-01-10T18:01:33.402ZLoop: 71 Downstairs dump after verify (and repair):
153692024-01-10T18:01:33.424ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153702024-01-10T18:01:33.424Z 0 000-019 141 141 141 253 253 253 F F F
153712024-01-10T18:01:33.424Z 1 020-039 141 141 141 253 253 253 F F F
153722024-01-10T18:01:33.424Z 2 040-059 134 134 134 241 241 241 F F F
153732024-01-10T18:01:33.424Z 3 060-079 138 138 138 249 249 249 F F F
153742024-01-10T18:01:33.425Z 4 080-099 140 140 140 252 252 252 F F F
153752024-01-10T18:01:33.425Z 5 100-119 140 140 140 252 252 252 F F F
153762024-01-10T18:01:33.425Z 6 120-139 142 142 142 254 254 254 F F F
153772024-01-10T18:01:33.425Z 7 140-159 140 140 140 252 252 252 F F F
153782024-01-10T18:01:33.428Z 8 160-179 140 140 140 252 252 252 F F F
153792024-01-10T18:01:33.428Z 9 180-199 140 140 140 252 252 252 F F F
153802024-01-10T18:01:33.428Z 10 200-219 135 135 135 243 243 243 F F F
153812024-01-10T18:01:33.429Z 11 220-239 135 135 135 243 243 243 F F F
153822024-01-10T18:01:33.429Z 12 240-259 139 139 139 250 250 250 F F F
153832024-01-10T18:01:33.429Z 13 260-279 133 133 133 240 240 240 F F F
153842024-01-10T18:01:33.429Z 14 280-299 140 140 140 252 252 252 F F F
153852024-01-10T18:01:33.429Z 15 300-319 142 142 142 254 254 254 F F F
153862024-01-10T18:01:33.429Z 16 320-339 140 140 140 252 252 252 F F F
153872024-01-10T18:01:33.429Z 17 340-359 142 142 142 254 254 254 F F F
153882024-01-10T18:01:33.429Z 18 360-379 143 143 143 256 256 256 F F F
153892024-01-10T18:01:33.429Z 19 380-399 143 143 143 256 256 256 F F F
153902024-01-10T18:01:33.429Z 20 400-419 126 126 126 230 230 230 F F F
153912024-01-10T18:01:33.429Z 21 420-439 141 141 141 253 253 253 F F F
153922024-01-10T18:01:33.429Z 22 440-459 143 143 143 256 256 256 F F F
153932024-01-10T18:01:33.429Z 23 460-479 133 133 133 240 240 240 F F F
153942024-01-10T18:01:33.429Z 24 480-499 136 136 136 244 244 244 F F F
153952024-01-10T18:01:33.429Z 25 500-519 143 143 143 256 256 256 F F F
153962024-01-10T18:01:33.429Z 26 520-539 142 142 142 255 255 255 F F F
153972024-01-10T18:01:33.429Z 27 540-559 143 143 143 256 256 256 F F F
153982024-01-10T18:01:33.430Z 28 560-579 143 143 143 256 256 256 F F F
153992024-01-10T18:01:33.430Z 29 580-599 143 143 143 256 256 256 F F F
154002024-01-10T18:01:33.430ZMax gen: 143, Max flush: 256
154012024-01-10T18:01:33.433Z
154022024-01-10T18:01:33.434ZBegin loop 72
154032024-01-10T18:01:33.434ZDownstairs to restart: 0
154042024-01-10T18:01:33.459Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.465541084Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734}
154052024-01-10T18:01:33.460Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.466078036Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734}
154062024-01-10T18:01:33.464Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.466150935Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734}
154072024-01-10T18:01:33.464Z{"msg":"Crucible 21458f33-9f31-4919-bacd-9b355b219ef4 has session id: f4a00869-3f02-4eee-9cc0-f4b54194c280","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.466322431Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154082024-01-10T18:01:33.464Z{Crucible runtime is spawned
154092024-01-10T18:01:33.464Z"msg":"Upstairs opts: Upstairs UUID: 21458f33-9f31-4919-bacd-9b355b219ef4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.466435482Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154102024-01-10T18:01:33.464Z{"msg":"Crucible stats registered with UUID: 21458f33-9f31-4919-bacd-9b355b219ef4","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.466510453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154112024-01-10T18:01:33.464Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.46655014Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734}
154122024-01-10T18:01:33.464Z{"msg":"21458f33-9f31-4919-bacd-9b355b219ef4 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.466712594Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154132024-01-10T18:01:33.464Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.466840424Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"0","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154142024-01-10T18:01:33.464Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.466886399Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"0","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154152024-01-10T18:01:33.464Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.466927319Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"1","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154162024-01-10T18:01:33.464Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.466985427Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"1","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154172024-01-10T18:01:33.464Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.467030483Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"2","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154182024-01-10T18:01:33.464Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.467209508Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"2","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154192024-01-10T18:01:33.464Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.467290859Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154202024-01-10T18:01:33.464Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.467336363Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154212024-01-10T18:01:33.464Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.467383746Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154222024-01-10T18:01:33.464Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.46746313Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154232024-01-10T18:01:33.464Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.467738688Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154242024-01-10T18:01:33.465Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.467825951Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154252024-01-10T18:01:33.465Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.46847325Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"2","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154262024-01-10T18:01:33.465Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.468586213Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"2","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154272024-01-10T18:01:33.465Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.468833327Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"1","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154282024-01-10T18:01:33.465Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.468916275Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"1","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154292024-01-10T18:01:33.465Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.469478818Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"2","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154302024-01-10T18:01:33.465Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.469670031Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"1","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154312024-01-10T18:01:33.466Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.469878914Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"0","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154322024-01-10T18:01:33.466Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.469930586Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"0","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154332024-01-10T18:01:33.466Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.470307486Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"2","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154342024-01-10T18:01:33.470Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.470379848Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"2","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154352024-01-10T18:01:33.470Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.47057775Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154362024-01-10T18:01:33.470Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.470637735Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"1","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154372024-01-10T18:01:33.470Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.470664445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"1","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154382024-01-10T18:01:33.470Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.470724787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154392024-01-10T18:01:33.470Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.470953596Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"0","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154402024-01-10T18:01:33.470Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.471614273Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"0","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154412024-01-10T18:01:33.470Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.471688393Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"0","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154422024-01-10T18:01:33.470Z{"msg":"[0]R flush_numbers[0..12]: [253, 253, 241, 249, 252, 252, 254, 252, 252, 252, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.471720432Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154432024-01-10T18:01:33.470Z{"msg":"[0]R generation[0..12]: [141, 141, 134, 138, 140, 140, 142, 140, 140, 140, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.471764989Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154442024-01-10T18:01:33.471Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.471790519Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154452024-01-10T18:01:33.471Z{"msg":"[1]R flush_numbers[0..12]: [253, 253, 241, 249, 252, 252, 254, 252, 252, 252, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.471831964Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154462024-01-10T18:01:33.471Z{"msg":"[1]R generation[0..12]: [141, 141, 134, 138, 140, 140, 142, 140, 140, 140, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.471856237Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154472024-01-10T18:01:33.471Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
154482024-01-10T18:01:33.471Z2024-01-10T18:01:33.471897955Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154492024-01-10T18:01:33.471Z{"msg":"[2]R flush_numbers[0..12]: [253, 253, 241, 249, 252, 252, 254, 252, 252, 252, 243, 243]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.471926219Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154502024-01-10T18:01:33.471Z{"msg":"[2]R generation[0..12]: [141, 141, 134, 138, 140, 140, 142, 140, 140, 140, 135, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.471967447Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154512024-01-10T18:01:33.471Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.471992192Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154522024-01-10T18:01:33.471Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.472032486Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154532024-01-10T18:01:33.471Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.472055824Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154542024-01-10T18:01:33.471Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.472141768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154552024-01-10T18:01:33.471Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.472182716Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"":"downstairs","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154562024-01-10T18:01:33.471Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.472242268Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154572024-01-10T18:01:33.471Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.4722826Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"0","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154582024-01-10T18:01:33.471Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.472347085Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"1","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154592024-01-10T18:01:33.471Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.472390858Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"client":"2","":"downstairs","session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154602024-01-10T18:01:33.471Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.47244963Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154612024-01-10T18:01:33.471Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.472474328Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154622024-01-10T18:01:33.471Z{"msg":"21458f33-9f31-4919-bacd-9b355b219ef4 is now active with session: f4a00869-3f02-4eee-9cc0-f4b54194c280","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.47249587Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154632024-01-10T18:01:33.471Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
154642024-01-10T18:01:33.471Z,"time":"2024-01-10T18:01:33.472567729Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734,"session_id":"f4a00869-3f02-4eee-9cc0-f4b54194c280"}
154652024-01-10T18:01:33.471Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:33.472605638Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1734}
154662024-01-10T18:01:33.471ZLoading write count information from file "/tmp/test_repair_verify.data"
154672024-01-10T18:01:33.472ZRun Repair workload
154682024-01-10T18:01:33.472Z01/30 Write block 435 len 16384 data: 10 10 10 10
154692024-01-10T18:01:33.472Z02/30 Read block 13 len 28672
154702024-01-10T18:01:33.475Z03/30 Read block 403 len 28672
154712024-01-10T18:01:33.475Z04/30 Read block 94 len 28672
154722024-01-10T18:01:33.478Z05/30 Write block 68 len 24576 data: 11 9 10 7 6 6
154732024-01-10T18:01:33.497Z06/30 Read block 388 len 32768
154742024-01-10T18:01:33.497Z07/30 Read block 509 len 8192
154752024-01-10T18:01:33.497Z08/30 Flush
154762024-01-10T18:01:33.497Z09/30 Write block 565 len 40960 data: 11 7 9 10 10 9 9 7 8 12
154772024-01-10T18:01:33.498Z10/30 Write block 235 len 28672 data: 7 7 9 9 10 10 11
154782024-01-10T18:01:33.498Z11/30 Read block 198 len 28672
154792024-01-10T18:01:33.498Z12/30 Read block 569 len 40960
154802024-01-10T18:01:33.506Z13/30 Read block 560 len 20480
154812024-01-10T18:01:33.506Z14/30 Write block 6 len 24576 data: 8 7 7 9 10 11
154822024-01-10T18:01:33.506Z15/30 Write block 358 len 36864 data: 12 10 7 7 9 11 10 9 8
154832024-01-10T18:01:33.506Z16/30 Read block 14 len 32768
154842024-01-10T18:01:33.518Z17/30 Write block 167 len 40960 data: 15 13 13 11 8 8 8 9 8 7
154852024-01-10T18:01:33.518Z18/30 Read block 57 len 24576
154862024-01-10T18:01:33.518Z19/30 Write block 267 len 16384 data: 12 10 11 12
154872024-01-10T18:01:33.518Z20/30 Read block 494 len 36864
154882024-01-10T18:01:33.519Z21/30 Write block 205 len 28672 data: 9 8 9 10 11 10 11
154892024-01-10T18:01:33.519Z22/30 Flush
154902024-01-10T18:01:33.530Z23/30 Flush
154912024-01-10T18:01:33.533Z24/30 Write block 508 len 8192 data: 8 13
154922024-01-10T18:01:33.533Z25/30 Read block 351 len 8192
154932024-01-10T18:01:33.533Z26/30 Flush
154942024-01-10T18:01:33.533Z27/30 Write block 18 len 32768 data: 11 10 9 10 8 10 8 9
154952024-01-10T18:01:33.534Z28/30 Read block 482 len 8192
154962024-01-10T18:01:33.543Z29/30 Write block 80 len 8192 data: 9 10
154972024-01-10T18:01:33.543Z30/30 Write block 443 len 36864 data: 8 8 12 15 15 13 13 13 11
154982024-01-10T18:01:33.544Z----------------------------------------------------------------
154992024-01-10T18:01:33.544Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
155002024-01-10T18:01:33.544ZStates: Active Active Active
155012024-01-10T18:01:33.544ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155022024-01-10T18:01:33.544Z 1 Acked 1000 Write 4 Sent Done Done false
155032024-01-10T18:01:33.544Z 2 Acked 1001 Read 7 Sent Done Done false
155042024-01-10T18:01:33.544Z 3 Acked 1002 Read 7 Sent Done Done false
155052024-01-10T18:01:33.544Z 4 Acked 1003 Read 7 Sent Done Done false
155062024-01-10T18:01:33.544Z 5 Acked 1004 Write 6 Sent Done Done false
155072024-01-10T18:01:33.544Z 6 Acked 1005 Read 8 Sent Done Done false
155082024-01-10T18:01:33.544Z 7 Acked 1006 Read 2 Sent Done Done false
155092024-01-10T18:01:33.545Z 8 Acked 1007 Flush 0 Sent Done Done false
155102024-01-10T18:01:33.545Z 9 Acked 1008 Write 10 Sent Done Done false
155112024-01-10T18:01:33.545Z 10 Acked 1009 Write 7 Sent Done Done false
155122024-01-10T18:01:33.545Z 11 Acked 1010 Read 7 Sent Done Done false
155132024-01-10T18:01:33.545Z 12 Acked 1011 Read 10 Sent Done Done false
155142024-01-10T18:01:33.545Z 13 Acked 1012 Read 5 Sent Done Done false
155152024-01-10T18:01:33.545Z 14 Acked 1013 Write 6 Sent Done Done false
155162024-01-10T18:01:33.546Z 15 Acked 1014 Write 9 Sent Done Done false
155172024-01-10T18:01:33.546Z 16 Acked 1015 Read 8 Sent Done Done false
155182024-01-10T18:01:33.546Z 17 Acked 1016 Write 10 Sent Done Done false
155192024-01-10T18:01:33.546Z 18 Acked 1017 Read 6 Sent Done Done false
155202024-01-10T18:01:33.546Z 19 Acked 1018 Write 4 Sent Done Done false
155212024-01-10T18:01:33.546Z 20 Acked 1019 Read 9 Sent Done Done false
155222024-01-10T18:01:33.546Z 21 Acked 1020 Write 7 Sent Done Done false
155232024-01-10T18:01:33.547Z 22 Acked 1021 Flush 0 Sent Done Done false
155242024-01-10T18:01:33.547Z 23 Acked 1022 Flush 0 Sent Done Done false
155252024-01-10T18:01:33.547Z 24 Acked 1023 Write 2 Sent Done Done false
155262024-01-10T18:01:33.547Z 25 Acked 1024 Read 2 Sent Done Done false
155272024-01-10T18:01:33.547Z 26 Acked 1025 Flush 0 Sent Done Done false
155282024-01-10T18:01:33.547Z 27 Acked 1026 Write 8 Sent Done Done false
155292024-01-10T18:01:33.548Z 28 Acked 1027 Read 2 Sent Done Done false
155302024-01-10T18:01:33.548Z 29 Acked 1028 Write 2 Sent Done Done false
155312024-01-10T18:01:33.548Z 30 Acked 1029 Write 9 Sent Sent Sent false
155322024-01-10T18:01:33.548Z STATES DS:0 DS:1 DS:2 TOTAL
155332024-01-10T18:01:33.548Z New 0 0 0 0
155342024-01-10T18:01:33.548Z Sent 30 1 1 32
155352024-01-10T18:01:33.548Z Done 0 29 29 58
155362024-01-10T18:01:33.549Z Skipped 0 0 0 0
155372024-01-10T18:01:33.549Z Error 0 0 0 0
155382024-01-10T18:01:33.549ZLast Flush: 0 1025 1025
155392024-01-10T18:01:33.550ZDownstairs last five completed:
155402024-01-10T18:01:33.550ZUpstairs last five completed: 30 29 28 27 26
155412024-01-10T18:01:33.550ZWrote out file "/tmp/test_repair_verify.data"
155422024-01-10T18:01:33.550Z
155432024-01-10T18:01:33.555ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155442024-01-10T18:01:33.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155452024-01-10T18:01:33.588Z 0 000-019 141 144 144 253 258 258 F T T <---
155462024-01-10T18:01:33.588Z 1 020-039 141 141 141 253 253 253 F T T <---
155472024-01-10T18:01:33.588Z 2 040-059 134 134 134 241 241 241 F F F
155482024-01-10T18:01:33.588Z 3 060-079 138 144 144 249 257 257 F F F <---
155492024-01-10T18:01:33.588Z 4 080-099 140 140 140 252 252 252 F T T <---
155502024-01-10T18:01:33.588Z 5 100-119 140 140 140 252 252 252 F F F
155512024-01-10T18:01:33.592Z 6 120-139 142 142 142 254 254 254 F F F
155522024-01-10T18:01:33.592Z 7 140-159 140 140 140 252 252 252 F F F
155532024-01-10T18:01:33.592Z 8 160-179 140 144 144 252 258 258 F F F <---
155542024-01-10T18:01:33.592Z 9 180-199 140 140 140 252 252 252 F F F
155552024-01-10T18:01:33.592Z 10 200-219 135 144 144 243 258 258 F F F <---
155562024-01-10T18:01:33.593Z 11 220-239 135 144 144 243 258 258 F F F <---
155572024-01-10T18:01:33.593Z 12 240-259 139 144 144 250 258 258 F F F <---
155582024-01-10T18:01:33.593Z 13 260-279 133 144 144 240 258 258 F F F <---
155592024-01-10T18:01:33.593Z 14 280-299 140 140 140 252 252 252 F F F
155602024-01-10T18:01:33.593Z 15 300-319 142 142 142 254 254 254 F F F
155612024-01-10T18:01:33.593Z 16 320-339 140 140 140 252 252 252 F F F
155622024-01-10T18:01:33.594Z 17 340-359 142 144 144 254 258 258 F F F <---
155632024-01-10T18:01:33.594Z 18 360-379 143 144 144 256 258 258 F F F <---
155642024-01-10T18:01:33.594Z 19 380-399 143 143 143 256 256 256 F F F
155652024-01-10T18:01:33.594Z 20 400-419 126 126 126 230 230 230 F F F
155662024-01-10T18:01:33.594Z 21 420-439 141 144 144 253 257 257 F F F <---
155672024-01-10T18:01:33.594Z 22 440-459 143 143 143 256 256 256 F T T <---
155682024-01-10T18:01:33.594Z 23 460-479 133 133 133 240 240 240 F F F
155692024-01-10T18:01:33.594Z 24 480-499 136 136 136 244 244 244 F F F
155702024-01-10T18:01:33.594Z 25 500-519 143 144 144 256 260 260 F F F <---
155712024-01-10T18:01:33.594Z 26 520-539 142 142 142 255 255 255 F F F
155722024-01-10T18:01:33.594Z 27 540-559 143 143 143 256 256 256 F F F
155732024-01-10T18:01:33.594Z 28 560-579 143 144 144 256 258 258 F F F <---
155742024-01-10T18:01:33.594Z 29 580-599 143 143 143 256 256 256 F F F
155752024-01-10T18:01:33.595ZMax gen: 144, Max flush: 260
155762024-01-10T18:01:33.595ZError: Difference in extent metadata found!
155772024-01-10T18:01:33.595ZA Difference in extent metadata is expected here
155782024-01-10T18:01:33.595ZOn loop 72
155792024-01-10T18:01:33.595Z
155802024-01-10T18:01:33.595ZVerifying data now
155812024-01-10T18:01:39.028ZNo change after verify
155822024-01-10T18:01:39.028ZLoop: 72 Downstairs dump after verify (and repair):
155832024-01-10T18:01:39.051ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155842024-01-10T18:01:39.052Z 0 000-019 145 145 145 261 261 261 F F F
155852024-01-10T18:01:39.053Z 1 020-039 145 145 145 261 261 261 F F F
155862024-01-10T18:01:39.053Z 2 040-059 134 134 134 241 241 241 F F F
155872024-01-10T18:01:39.053Z 3 060-079 144 144 144 257 257 257 F F F
155882024-01-10T18:01:39.065Z 4 080-099 145 145 145 261 261 261 F F F
155892024-01-10T18:01:39.065Z 5 100-119 140 140 140 252 252 252 F F F
155902024-01-10T18:01:39.065Z 6 120-139 142 142 142 254 254 254 F F F
155912024-01-10T18:01:39.065Z 7 140-159 140 140 140 252 252 252 F F F
155922024-01-10T18:01:39.065Z 8 160-179 144 144 144 258 258 258 F F F
155932024-01-10T18:01:39.066Z 9 180-199 140 140 140 252 252 252 F F F
155942024-01-10T18:01:39.066Z 10 200-219 144 144 144 258 258 258 F F F
155952024-01-10T18:01:39.066Z 11 220-239 144 144 144 258 258 258 F F F
155962024-01-10T18:01:39.066Z 12 240-259 144 144 144 258 258 258 F F F
155972024-01-10T18:01:39.066Z 13 260-279 144 144 144 258 258 258 F F F
155982024-01-10T18:01:39.066Z 14 280-299 140 140 140 252 252 252 F F F
155992024-01-10T18:01:39.066Z 15 300-319 142 142 142 254 254 254 F F F
156002024-01-10T18:01:39.066Z 16 320-339 140 140 140 252 252 252 F F F
156012024-01-10T18:01:39.066Z 17 340-359 144 144 144 258 258 258 F F F
156022024-01-10T18:01:39.066Z 18 360-379 144 144 144 258 258 258 F F F
156032024-01-10T18:01:39.066Z 19 380-399 143 143 143 256 256 256 F F F
156042024-01-10T18:01:39.066Z 20 400-419 126 126 126 230 230 230 F F F
156052024-01-10T18:01:39.066Z 21 420-439 144 144 144 257 257 257 F F F
156062024-01-10T18:01:39.066Z 22 440-459 145 145 145 261 261 261 F F F
156072024-01-10T18:01:39.067Z 23 460-479 133 133 133 240 240 240 F F F
156082024-01-10T18:01:39.067Z 24 480-499 136 136 136 244 244 244 F F F
156092024-01-10T18:01:39.067Z 25 500-519 144 144 144 260 260 260 F F F
156102024-01-10T18:01:39.067Z 26 520-539 142 142 142 255 255 255 F F F
156112024-01-10T18:01:39.068Z 27 540-559 143 143 143 256 256 256 F F F
156122024-01-10T18:01:39.068Z 28 560-579 144 144 144 258 258 258 F F F
156132024-01-10T18:01:39.068Z 29 580-599 143 143 143 256 256 256 F F F
156142024-01-10T18:01:39.068ZMax gen: 145, Max flush: 261
156152024-01-10T18:01:39.068Z
156162024-01-10T18:01:39.068ZBegin loop 73
156172024-01-10T18:01:39.068ZDownstairs to restart: 1
156182024-01-10T18:01:39.084Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.091398445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742}
156192024-01-10T18:01:39.085Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.091908873Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742}
156202024-01-10T18:01:39.089Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.091979977Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742}
156212024-01-10T18:01:39.089Z{"msg":"Crucible 3b31128b-8dbc-4d2d-a7fa-34515a8101d4 has session id: 42e97fd7-a549-4623-b2d7-45a5d2171649","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.092053796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156222024-01-10T18:01:39.090Z{"msg":"Crucible runtime is spawned
156232024-01-10T18:01:39.090ZUpstairs opts: Upstairs UUID: 3b31128b-8dbc-4d2d-a7fa-34515a8101d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.092308872Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156242024-01-10T18:01:39.090Z{"msg":"Crucible stats registered with UUID: 3b31128b-8dbc-4d2d-a7fa-34515a8101d4","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.092448914Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156252024-01-10T18:01:39.090Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.092514035Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742}
156262024-01-10T18:01:39.091Z{"msg":"3b31128b-8dbc-4d2d-a7fa-34515a8101d4 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.09255238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156272024-01-10T18:01:39.091Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.092609584Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"0","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156282024-01-10T18:01:39.091Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.092703452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"0","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156292024-01-10T18:01:39.091Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.09274751Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"1","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156302024-01-10T18:01:39.091Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.092803595Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"1","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156312024-01-10T18:01:39.091Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.092897039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"2","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156322024-01-10T18:01:39.091Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.09294298Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"2","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156332024-01-10T18:01:39.091Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.092984913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156342024-01-10T18:01:39.091Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.093029151Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156352024-01-10T18:01:39.091Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.093099297Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156362024-01-10T18:01:39.092Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.093143287Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156372024-01-10T18:01:39.092Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.093184489Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156382024-01-10T18:01:39.092Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.093226953Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156392024-01-10T18:01:39.092Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.094461309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"2","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156402024-01-10T18:01:39.092Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.094571073Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"2","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156412024-01-10T18:01:39.092Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.094616988Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"0","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156422024-01-10T18:01:39.092Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.094714804Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"0","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156432024-01-10T18:01:39.092Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.095586124Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"2","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156442024-01-10T18:01:39.092Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.095743457Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"0","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156452024-01-10T18:01:39.092Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.095808947Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"1","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156462024-01-10T18:01:39.092Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.095852965Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"1","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156472024-01-10T18:01:39.100Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.096551348Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"0","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156482024-01-10T18:01:39.100Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.096614079Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"0","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156492024-01-10T18:01:39.100Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.096639207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156502024-01-10T18:01:39.100Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.096766116Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"2","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156512024-01-10T18:01:39.100Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.096818437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"2","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156522024-01-10T18:01:39.100Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.096843228Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156532024-01-10T18:01:39.100Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.097144381Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"1","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156542024-01-10T18:01:39.100Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.097812576Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"1","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156552024-01-10T18:01:39.100Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.097881875Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"1","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156562024-01-10T18:01:39.101Z{"msg":"[0]R flush_numbers[0..12]: [261, 261, 241, 257, 261, 252, 254, 252, 258, 252, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.097928912Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156572024-01-10T18:01:39.102Z{"msg":"[0]R generation[0..12]: [145, 145, 134, 144, 145, 140, 142, 140, 144, 140, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.09798457Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156582024-01-10T18:01:39.102Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098030905Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156592024-01-10T18:01:39.102Z{"msg":"[1]R flush_numbers[0..12]: [261, 261, 241, 257, 261, 252, 254, 252, 258, 252, 258, 258]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
156602024-01-10T18:01:39.102Z,"time":"2024-01-10T18:01:39.098085001Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156612024-01-10T18:01:39.102Z{"msg":"[1]R generation[0..12]: [145, 145, 134, 144, 145, 140, 142, 140, 144, 140, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098130398Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156622024-01-10T18:01:39.102Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.09818579Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156632024-01-10T18:01:39.102Z{"msg":"[2]R flush_numbers[0..12]: [261, 261, 241, 257, 261, 252, 254, 252, 258, 252, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.09823027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156642024-01-10T18:01:39.102Z{"msg":"[2]R generation[0..12]: [145, 145, 134, 144, 145, 140, 142, 140, 144, 140, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098293107Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156652024-01-10T18:01:39.102Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098338484Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156662024-01-10T18:01:39.103Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098394832Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156672024-01-10T18:01:39.103Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098437424Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156682024-01-10T18:01:39.103Z{"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098495851Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156692024-01-10T18:01:39.103Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098537026Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"":"downstairs","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156702024-01-10T18:01:39.103Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098595915Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156712024-01-10T18:01:39.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098633316Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"0","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156722024-01-10T18:01:39.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098689926Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"1","":"downstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156732024-01-10T18:01:39.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098731885Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"client":"2","":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
156742024-01-10T18:01:39.103Zdownstairs","session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156752024-01-10T18:01:39.103Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098832869Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156762024-01-10T18:01:39.103Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.09889586Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156772024-01-10T18:01:39.104Z{"msg":"3b31128b-8dbc-4d2d-a7fa-34515a8101d4 is now active with session: 42e97fd7-a549-4623-b2d7-45a5d2171649","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098938687Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156782024-01-10T18:01:39.104Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.098988113Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742,"session_id":"42e97fd7-a549-4623-b2d7-45a5d2171649"}
156792024-01-10T18:01:39.104Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:39.099026993Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1742}
156802024-01-10T18:01:39.104ZLoading write count information from file "/tmp/test_repair_verify.data"
156812024-01-10T18:01:39.104ZRun Repair workload
156822024-01-10T18:01:39.104Z01/30 Write block 460 len 4096 data: 11
156832024-01-10T18:01:39.104Z02/30 Flush
156842024-01-10T18:01:39.114Z03/30 Write block 119 len 28672 data: 13 12 13 13 10 7 7
156852024-01-10T18:01:39.114Z04/30 Read block 412 len 8192
156862024-01-10T18:01:39.114Z05/30 Read block 350 len 24576
156872024-01-10T18:01:39.114Z06/30 Write block 262 len 16384 data: 8 8 9 7
156882024-01-10T18:01:39.114Z07/30 Read block 334 len 24576
156892024-01-10T18:01:39.114Z08/30 Read block 201 len 40960
156902024-01-10T18:01:39.114Z09/30 Write block 75 len 40960 data: 12 12 12 8 10 10 11 10 13 9
156912024-01-10T18:01:39.114Z10/30 Read block 138 len 40960
156922024-01-10T18:01:39.118Z11/30 Write block 92 len 4096 data: 13
156932024-01-10T18:01:39.118Z12/30 Write block 474 len 40960 data: 7 6 6 6 4 6 7 7 7 9
156942024-01-10T18:01:39.118Z13/30 Write block 389 len 24576 data: 7 9 9 10 8 9
156952024-01-10T18:01:39.123Z14/30 Read block 220 len 12288
156962024-01-10T18:01:39.123Z15/30 Read block 123 len 32768
156972024-01-10T18:01:39.127Z16/30 Write block 280 len 24576 data: 11 10 9 10 11 13
156982024-01-10T18:01:39.127Z17/30 Write block 226 len 8192 data: 7 8
156992024-01-10T18:01:39.127Z18/30 Read block 349 len 16384
157002024-01-10T18:01:39.130Z19/30 Flush
157012024-01-10T18:01:39.134Z20/30 Write block 437 len 28672 data: 11 11 11 9 12 9 9
157022024-01-10T18:01:39.134Z21/30 Write block 461 len 4096 data: 9
157032024-01-10T18:01:39.134Z22/30 Write block 257 len 16384 data: 9 12 11 10
157042024-01-10T18:01:39.138Z23/30 Read block 71 len 28672
157052024-01-10T18:01:39.138Z24/30 Write block 44 len 8192 data: 10 10
157062024-01-10T18:01:39.146Z25/30 Flush
157072024-01-10T18:01:39.146Z26/30 Flush
157082024-01-10T18:01:39.146Z27/30 Write block 125 len 36864 data: 8 6 6 4 4 7 6 8 8
157092024-01-10T18:01:39.146Z28/30 Write block 206 len 12288 data: 9 10 11
157102024-01-10T18:01:39.152Z29/30 Write block 486 len 12288 data: 10 9 10
157112024-01-10T18:01:39.152Z30/30 Write block 177 len 24576 data: 9 9 9 12 10 10
157122024-01-10T18:01:39.152Z----------------------------------------------------------------
157132024-01-10T18:01:39.152Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
157142024-01-10T18:01:39.152ZStates: Active Active Active
157152024-01-10T18:01:39.152ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157162024-01-10T18:01:39.152Z 1 Acked 1000 Write 1 Done Done Done false
157172024-01-10T18:01:39.152Z 2 Acked 1001 Flush 0 Done Sent Done false
157182024-01-10T18:01:39.152Z 3 Acked 1002 Write 7 Done Sent Done false
157192024-01-10T18:01:39.152Z 4 Acked 1003 Read 2 Done Sent Done false
157202024-01-10T18:01:39.152Z 5 Acked 1004 Read 6 Done Sent Done false
157212024-01-10T18:01:39.152Z 6 Acked 1005 Write 4 Done Sent Done false
157222024-01-10T18:01:39.153Z 7 Acked 1006 Read 6 Done Sent Done false
157232024-01-10T18:01:39.153Z 8 Acked 1007 Read 10 Done Sent Done false
157242024-01-10T18:01:39.153Z 9 Acked 1008 Write 10 Done Sent Done false
157252024-01-10T18:01:39.153Z 10 Acked 1009 Read 10 Done Sent Done false
157262024-01-10T18:01:39.153Z 11 Acked 1010 Write 1 Done Sent Done false
157272024-01-10T18:01:39.153Z 12 Acked 1011 Write 10 Done Sent Done false
157282024-01-10T18:01:39.154Z 13 Acked 1012 Write 6 Done Sent Done false
157292024-01-10T18:01:39.154Z 14 Acked 1013 Read 3 Done Sent Done false
157302024-01-10T18:01:39.154Z 15 Acked 1014 Read 8 Done Sent Done false
157312024-01-10T18:01:39.154Z 16 Acked 1015 Write 6 Done Sent Done false
157322024-01-10T18:01:39.154Z 17 Acked 1016 Write 2 Done Sent Done false
157332024-01-10T18:01:39.154Z 18 Acked 1017 Read 4 Done Sent Done false
157342024-01-10T18:01:39.154Z 19 Acked 1018 Flush 0 Done Sent Done false
157352024-01-10T18:01:39.154Z 20 Acked 1019 Write 7 Done Sent Done false
157362024-01-10T18:01:39.154Z 21 Acked 1020 Write 1 Done Sent Done false
157372024-01-10T18:01:39.154Z 22 Acked 1021 Write 4 Done Sent Done false
157382024-01-10T18:01:39.154Z 23 Acked 1022 Read 7 Done Sent Done false
157392024-01-10T18:01:39.154Z 24 Acked 1023 Write 2 Done Sent Done false
157402024-01-10T18:01:39.154Z 25 Acked 1024 Flush 0 Done Sent Done false
157412024-01-10T18:01:39.154Z 26 Acked 1025 Flush 0 Done Sent Done false
157422024-01-10T18:01:39.154Z 27 Acked 1026 Write 9 Done Sent Done false
157432024-01-10T18:01:39.154Z 28 Acked 1027 Write 3 Done Sent Done false
157442024-01-10T18:01:39.154Z 29 Acked 1028 Write 3 Done Sent Done false
157452024-01-10T18:01:39.154Z 30 Acked 1029 Write 6 Sent Sent Sent false
157462024-01-10T18:01:39.154Z STATES DS:0 DS:1 DS:2 TOTAL
157472024-01-10T18:01:39.154Z New 0 0 0 0
157482024-01-10T18:01:39.154Z Sent 1 29 1 31
157492024-01-10T18:01:39.154Z Done 29 1 29 59
157502024-01-10T18:01:39.154Z Skipped 0 0 0 0
157512024-01-10T18:01:39.154Z Error 0 0 0 0
157522024-01-10T18:01:39.154ZLast Flush: 1025 0 1025
157532024-01-10T18:01:39.155ZDownstairs last five completed:
157542024-01-10T18:01:39.155ZUpstairs last five completed: 30 29 28 27 26
157552024-01-10T18:01:39.155ZWrote out file "/tmp/test_repair_verify.data"
157562024-01-10T18:01:39.158Z
157572024-01-10T18:01:39.161ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
157582024-01-10T18:01:39.186ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157592024-01-10T18:01:39.186Z 0 000-019 145 145 145 261 261 261 F F F
157602024-01-10T18:01:39.187Z 1 020-039 145 145 145 261 261 261 F F F
157612024-01-10T18:01:39.187Z 2 040-059 146 134 146 264 241 264 F F F <---
157622024-01-10T18:01:39.187Z 3 060-079 146 144 146 263 257 263 F F F <---
157632024-01-10T18:01:39.187Z 4 080-099 146 145 146 263 261 263 F F F <---
157642024-01-10T18:01:39.187Z 5 100-119 146 140 146 263 252 263 F F F <---
157652024-01-10T18:01:39.191Z 6 120-139 146 142 146 263 254 263 T F T <---
157662024-01-10T18:01:39.192Z 7 140-159 140 140 140 252 252 252 F F F
157672024-01-10T18:01:39.192Z 8 160-179 144 144 144 258 258 258 T F T <---
157682024-01-10T18:01:39.192Z 9 180-199 140 140 140 252 252 252 T F T <---
157692024-01-10T18:01:39.192Z 10 200-219 144 144 144 258 258 258 T F T <---
157702024-01-10T18:01:39.194Z 11 220-239 146 144 146 263 258 263 F F F <---
157712024-01-10T18:01:39.194Z 12 240-259 146 144 146 264 258 264 F F F <---
157722024-01-10T18:01:39.194Z 13 260-279 146 144 146 264 258 264 F F F <---
157732024-01-10T18:01:39.194Z 14 280-299 146 140 146 263 252 263 F F F <---
157742024-01-10T18:01:39.195Z 15 300-319 142 142 142 254 254 254 F F F
157752024-01-10T18:01:39.195Z 16 320-339 140 140 140 252 252 252 F F F
157762024-01-10T18:01:39.195Z 17 340-359 144 144 144 258 258 258 F F F
157772024-01-10T18:01:39.195Z 18 360-379 144 144 144 258 258 258 F F F
157782024-01-10T18:01:39.195Z 19 380-399 146 143 146 263 256 263 F F F <---
157792024-01-10T18:01:39.195Z 20 400-419 126 126 126 230 230 230 F F F
157802024-01-10T18:01:39.195Z 21 420-439 146 144 146 264 257 264 F F F <---
157812024-01-10T18:01:39.195Z 22 440-459 146 145 146 264 261 264 F F F <---
157822024-01-10T18:01:39.195Z 23 460-479 146 133 146 264 240 264 F T F <---
157832024-01-10T18:01:39.195Z 24 480-499 146 136 146 263 244 263 T F T <---
157842024-01-10T18:01:39.195Z 25 500-519 144 144 144 260 260 260 F F F
157852024-01-10T18:01:39.195Z 26 520-539 142 142 142 255 255 255 F F F
157862024-01-10T18:01:39.195Z 27 540-559 143 143 143 256 256 256 F F F
157872024-01-10T18:01:39.195Z 28 560-579 144 144 144 258 258 258 F F F
157882024-01-10T18:01:39.195Z 29 580-599 143 143 143 256 256 256 F F F
157892024-01-10T18:01:39.195ZMax gen: 146, Max flush: 264
157902024-01-10T18:01:39.195ZError: Difference in extent metadata found!
157912024-01-10T18:01:39.195ZA Difference in extent metadata is expected here
157922024-01-10T18:01:39.195ZOn loop 73
157932024-01-10T18:01:39.195Z
157942024-01-10T18:01:39.199ZVerifying data now
157952024-01-10T18:01:40.763ZNo change after verify
157962024-01-10T18:01:40.769ZLoop: 73 Downstairs dump after verify (and repair):
157972024-01-10T18:01:40.784ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157982024-01-10T18:01:40.785Z 0 000-019 145 145 145 261 261 261 F F F
157992024-01-10T18:01:40.785Z 1 020-039 145 145 145 261 261 261 F F F
158002024-01-10T18:01:40.785Z 2 040-059 146 146 146 264 264 264 F F F
158012024-01-10T18:01:40.785Z 3 060-079 146 146 146 263 263 263 F F F
158022024-01-10T18:01:40.785Z 4 080-099 146 146 146 263 263 263 F F F
158032024-01-10T18:01:40.785Z 5 100-119 146 146 146 263 263 263 F F F
158042024-01-10T18:01:40.785Z 6 120-139 147 147 147 265 265 265 F F F
158052024-01-10T18:01:40.792Z 7 140-159 140 140 140 252 252 252 F F F
158062024-01-10T18:01:40.792Z 8 160-179 147 147 147 265 265 265 F F F
158072024-01-10T18:01:40.792Z 9 180-199 147 147 147 265 265 265 F F F
158082024-01-10T18:01:40.792Z 10 200-219 147 147 147 265 265 265 F F F
158092024-01-10T18:01:40.793Z 11 220-239 146 146 146 263 263 263 F F F
158102024-01-10T18:01:40.793Z 12 240-259 146 146 146 264 264 264 F F F
158112024-01-10T18:01:40.793Z 13 260-279 146 146 146 264 264 264 F F F
158122024-01-10T18:01:40.793Z 14 280-299 146 146 146 263 263 263 F F F
158132024-01-10T18:01:40.793Z 15 300-319 142 142 142 254 254 254 F F F
158142024-01-10T18:01:40.793Z 16 320-339 140 140 140 252 252 252 F F F
158152024-01-10T18:01:40.793Z 17 340-359 144 144 144 258 258 258 F F F
158162024-01-10T18:01:40.793Z 18 360-379 144 144 144 258 258 258 F F F
158172024-01-10T18:01:40.793Z 19 380-399 146 146 146 263 263 263 F F F
158182024-01-10T18:01:40.793Z 20 400-419 126 126 126 230 230 230 F F F
158192024-01-10T18:01:40.794Z 21 420-439 146 146 146 264 264 264 F F F
158202024-01-10T18:01:40.794Z 22 440-459 146 146 146 264 264 264 F F F
158212024-01-10T18:01:40.794Z 23 460-479 146 146 146 264 264 264 F F F
158222024-01-10T18:01:40.795Z 24 480-499 147 147 147 265 265 265 F F F
158232024-01-10T18:01:40.795Z 25 500-519 144 144 144 260 260 260 F F F
158242024-01-10T18:01:40.795Z 26 520-539 142 142 142 255 255 255 F F F
158252024-01-10T18:01:40.796Z 27 540-559 143 143 143 256 256 256 F F F
158262024-01-10T18:01:40.796Z 28 560-579 144 144 144 258 258 258 F F F
158272024-01-10T18:01:40.796Z 29 580-599 143 143 143 256 256 256 F F F
158282024-01-10T18:01:40.796ZMax gen: 147, Max flush: 265
158292024-01-10T18:01:40.796Z
158302024-01-10T18:01:40.796ZBegin loop 74
158312024-01-10T18:01:40.796ZDownstairs to restart: 1
158322024-01-10T18:01:40.817Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.822545927Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750}
158332024-01-10T18:01:40.828Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.82305286Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750}
158342024-01-10T18:01:40.828Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823133913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750}
158352024-01-10T18:01:40.828Z{"msg":"Crucible bc807fb2-6c1a-436d-9ec3-1218ef93b2b0 has session id: 8eecf723-b592-4902-8cbf-5b2c2149692b","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.82319613Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158362024-01-10T18:01:40.828ZCrucible runtime is spawned
158372024-01-10T18:01:40.828Z{"msg":"Upstairs opts: Upstairs UUID: bc807fb2-6c1a-436d-9ec3-1218ef93b2b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823437609Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158382024-01-10T18:01:40.829Z{"msg":"Crucible stats registered with UUID: bc807fb2-6c1a-436d-9ec3-1218ef93b2b0","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823507929Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158392024-01-10T18:01:40.829Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823549926Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750}
158402024-01-10T18:01:40.829Z{"msg":"bc807fb2-6c1a-436d-9ec3-1218ef93b2b0 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823633723Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158412024-01-10T18:01:40.829Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823723446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"0","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158422024-01-10T18:01:40.829Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823771804Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"0","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158432024-01-10T18:01:40.829Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823812376Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"1","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158442024-01-10T18:01:40.829Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823854373Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"1","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158452024-01-10T18:01:40.829Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823895439Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"2","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158462024-01-10T18:01:40.829Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.823934861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"2","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158472024-01-10T18:01:40.829Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.824073229Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158482024-01-10T18:01:40.829Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.824118421Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158492024-01-10T18:01:40.829Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.824156985Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158502024-01-10T18:01:40.829Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.824216644Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158512024-01-10T18:01:40.829Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.824260267Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158522024-01-10T18:01:40.830Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.824568447Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158532024-01-10T18:01:40.830Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.825335979Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"2","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158542024-01-10T18:01:40.830Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.825423185Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"2","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158552024-01-10T18:01:40.830Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.825471594Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"0","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158562024-01-10T18:01:40.830Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.825516446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"0","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158572024-01-10T18:01:40.830Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.826317863Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"1","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158582024-01-10T18:01:40.831Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.826392543Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"1","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158592024-01-10T18:01:40.831Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.826458213Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"2","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158602024-01-10T18:01:40.831Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.82654615Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"0","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158612024-01-10T18:01:40.831Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.82720464Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"1","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158622024-01-10T18:01:40.832Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.827454251Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"2","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158632024-01-10T18:01:40.832Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.827519422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"2","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158642024-01-10T18:01:40.832Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.827546037Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158652024-01-10T18:01:40.832Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.82756667Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"0","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158662024-01-10T18:01:40.832Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.827589663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"0","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158672024-01-10T18:01:40.832Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.827626181Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158682024-01-10T18:01:40.846Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828049158Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"1","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158692024-01-10T18:01:40.846Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828110259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"1","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158702024-01-10T18:01:40.846Z{"msg":"[0]R flush_numbers[0..12]: [261, 261, 264, 263, 263, 263, 265, 252, 265, 265, 265, 263]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828137661Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158712024-01-10T18:01:40.847Z{"msg":"[0]R generation[0..12]: [145, 145, 146, 146, 146, 146, 147, 140, 147, 147, 147, 146]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828181995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158722024-01-10T18:01:40.847Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828207462Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158732024-01-10T18:01:40.847Z{"msg":"[1]R flush_numbers[0..12]: [261, 261, 264, 263, 263, 263, 265, 252, 265, 265, 265, 263]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828263043Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158742024-01-10T18:01:40.847Z{"msg":"[1]R generation[0..12]: [145, 145, 146, 146, 146, 146, 147, 140, 147, 147, 147, 146]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828287878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158752024-01-10T18:01:40.847Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
158762024-01-10T18:01:40.847Z,"time":"2024-01-10T18:01:40.82831125Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158772024-01-10T18:01:40.847Z{"msg":"[2]R flush_numbers[0..12]: [261, 261, 264, 263, 263, 263, 265, 252, 265, 265, 265, 263]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828337034Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158782024-01-10T18:01:40.848Z{"msg":"[2]R generation[0..12]: [145, 145, 146, 146, 146, 146, 147, 140, 147, 147, 147, 146]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.82838196Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158792024-01-10T18:01:40.848Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828408514Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158802024-01-10T18:01:40.849Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828462006Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158812024-01-10T18:01:40.849Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828485734Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158822024-01-10T18:01:40.849Z{"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828507082Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158832024-01-10T18:01:40.849Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828540281Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"":"downstairs","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158842024-01-10T18:01:40.849Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828577326Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158852024-01-10T18:01:40.849Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828636714Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"0","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158862024-01-10T18:01:40.849Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.82866204Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"1","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158872024-01-10T18:01:40.849Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828728812Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"client":"2","":"downstairs","session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158882024-01-10T18:01:40.850Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828782369Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158892024-01-10T18:01:40.850Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828818138Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158902024-01-10T18:01:40.850Z{"msg":"bc807fb2-6c1a-436d-9ec3-1218ef93b2b0 is now active with session: 8eecf723-b592-4902-8cbf-5b2c2149692b","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828858109Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158912024-01-10T18:01:40.850Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828894947Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750,"session_id":"8eecf723-b592-4902-8cbf-5b2c2149692b"}
158922024-01-10T18:01:40.851Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:40.828931787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1750}
158932024-01-10T18:01:40.851ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
158942024-01-10T18:01:40.851ZLoading write count information from file "/tmp/test_repair_verify.data"
158952024-01-10T18:01:40.851ZRun Repair workload
158962024-01-10T18:01:40.851Z01/30 Write block 357 len 24576 data: 10 13 11 8 8 10
158972024-01-10T18:01:40.851Z02/30 Read block 268 len 24576
158982024-01-10T18:01:40.851Z03/30 Write block 335 len 28672 data: 11 11 11 11 14 11 11
158992024-01-10T18:01:40.851Z04/30 Read block 330 len 32768
159002024-01-10T18:01:40.851Z05/30 Read block 221 len 32768
159012024-01-10T18:01:40.851Z06/30 Read block 182 len 24576
159022024-01-10T18:01:40.861Z07/30 Write block 417 len 20480 data: 11 11 16 19 17
159032024-01-10T18:01:40.862Z08/30 Read block 459 len 20480
159042024-01-10T18:01:40.862Z09/30 Read block 165 len 24576
159052024-01-10T18:01:40.862Z10/30 Write block 541 len 8192 data: 15 10
159062024-01-10T18:01:40.862Z11/30 Write block 447 len 24576 data: 16 14 14 14 12 13
159072024-01-10T18:01:40.862Z12/30 Read block 408 len 24576
159082024-01-10T18:01:40.862Z13/30 Read block 126 len 24576
159092024-01-10T18:01:40.862Z14/30 Read block 496 len 16384
159102024-01-10T18:01:40.862Z15/30 Write block 259 len 12288 data: 12 11 10
159112024-01-10T18:01:40.863Z16/30 Write block 428 len 8192 data: 10 10
159122024-01-10T18:01:40.863Z17/30 Read block 17 len 16384
159132024-01-10T18:01:40.863Z18/30 Read block 123 len 8192
159142024-01-10T18:01:40.863Z19/30 Flush
159152024-01-10T18:01:40.877Z20/30 Write block 116 len 36864 data: 12 12 11 14 13 14 14 11 8
159162024-01-10T18:01:40.877Z21/30 Write block 28 len 12288 data: 10 11 15
159172024-01-10T18:01:40.877Z22/30 Read block 479 len 8192
159182024-01-10T18:01:40.877Z23/30 Read block 445 len 16384
159192024-01-10T18:01:40.877Z24/30 Read block 420 len 28672
159202024-01-10T18:01:40.877Z25/30 Write block 158 len 32768 data: 9 8 9 13 14 12 14 15
159212024-01-10T18:01:40.877Z26/30 Write block 419 len 40960 data: 17 20 18 16 15 14 13 12 10 11
159222024-01-10T18:01:40.877Z27/30 Read block 551 len 28672
159232024-01-10T18:01:40.877Z28/30 Read block 248 len 4096
159242024-01-10T18:01:40.882Z29/30 Write block 221 len 36864 data: 11 10 8 6 8 8 9 9 9
159252024-01-10T18:01:40.882Z30/30 Read block 103 len 20480
159262024-01-10T18:01:40.882Z----------------------------------------------------------------
159272024-01-10T18:01:40.882Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
159282024-01-10T18:01:40.882ZStates: Active Active Active
159292024-01-10T18:01:40.882ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159302024-01-10T18:01:40.882Z 1 Acked 1000 Write 6 Done Done Done false
159312024-01-10T18:01:40.882Z 2 Acked 1001 Read 6 Done Done Done false
159322024-01-10T18:01:40.882Z 3 Acked 1002 Write 7 Done Sent Done false
159332024-01-10T18:01:40.882Z 4 Acked 1003 Read 8 Done Sent Done false
159342024-01-10T18:01:40.882Z 5 Acked 1004 Read 8 Done Sent Done false
159352024-01-10T18:01:40.882Z 6 Acked 1005 Read 6 Done Sent Done false
159362024-01-10T18:01:40.882Z 7 Acked 1006 Write 5 Done Sent Done false
159372024-01-10T18:01:40.882Z 8 Acked 1007 Read 5 Done Sent Done false
159382024-01-10T18:01:40.882Z 9 Acked 1008 Read 6 Done Sent Done false
159392024-01-10T18:01:40.882Z 10 Acked 1009 Write 2 Done Sent Done false
159402024-01-10T18:01:40.882Z 11 Acked 1010 Write 6 Done Sent Done false
159412024-01-10T18:01:40.883Z 12 Acked 1011 Read 6 Done Sent Done false
159422024-01-10T18:01:40.883Z 13 Acked 1012 Read 6 Done Sent Done false
159432024-01-10T18:01:40.883Z 14 Acked 1013 Read 4 Done Sent Done false
159442024-01-10T18:01:40.883Z 15 Acked 1014 Write 3 Done Sent Done false
159452024-01-10T18:01:40.883Z 16 Acked 1015 Write 2 Done Sent Done false
159462024-01-10T18:01:40.883Z 17 Acked 1016 Read 4 Done Sent Done false
159472024-01-10T18:01:40.883Z 18 Acked 1017 Read 2 Done Sent Done false
159482024-01-10T18:01:40.883Z 19 Acked 1018 Flush 0 Done Sent Done false
159492024-01-10T18:01:40.883Z 20 Acked 1019 Write 9 Done Sent Done false
159502024-01-10T18:01:40.883Z 21 Acked 1020 Write 3 Done Sent Done false
159512024-01-10T18:01:40.883Z 22 Acked 1021 Read 2 Done Sent Done false
159522024-01-10T18:01:40.883Z 23 Acked 1022 Read 4 Done Sent Done false
159532024-01-10T18:01:40.883Z 24 Acked 1023 Read 7 Done Sent Done false
159542024-01-10T18:01:40.884Z 25 Acked 1024 Write 8 Done Sent Done false
159552024-01-10T18:01:40.884Z 26 Acked 1025 Write 10 Done Sent Done false
159562024-01-10T18:01:40.885Z 27 Acked 1026 Read 7 Done Sent Done false
159572024-01-10T18:01:40.885Z 28 Acked 1027 Read 1 Done Sent Done false
159582024-01-10T18:01:40.885Z 29 Acked 1028 Write 9 Done Sent Sent false
159592024-01-10T18:01:40.885Z 30 Acked 1029 Read 5 Done Sent Sent false
159602024-01-10T18:01:40.885Z STATES DS:0 DS:1 DS:2 TOTAL
159612024-01-10T18:01:40.885Z New 0 0 0 0
159622024-01-10T18:01:40.885Z Sent 0 28 2 30
159632024-01-10T18:01:40.885Z Done 30 2 28 60
159642024-01-10T18:01:40.885Z Skipped 0 0 0 0
159652024-01-10T18:01:40.885Z Error 0 0 0 0
159662024-01-10T18:01:40.885ZLast Flush: 1018 0 1018
159672024-01-10T18:01:40.885ZDownstairs last five completed:
159682024-01-10T18:01:40.886ZUpstairs last five completed: 30 29 28 27 26
159692024-01-10T18:01:40.886ZWrote out file "/tmp/test_repair_verify.data"
159702024-01-10T18:01:40.889Z
159712024-01-10T18:01:40.893ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
159722024-01-10T18:01:40.921ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159732024-01-10T18:01:40.921Z 0 000-019 145 145 145 261 261 261 F F F
159742024-01-10T18:01:40.921Z 1 020-039 145 145 145 261 261 261 T F T <---
159752024-01-10T18:01:40.921Z 2 040-059 146 146 146 264 264 264 F F F
159762024-01-10T18:01:40.921Z 3 060-079 146 146 146 263 263 263 F F F
159772024-01-10T18:01:40.921Z 4 080-099 146 146 146 263 263 263 F F F
159782024-01-10T18:01:40.922Z 5 100-119 146 146 146 263 263 263 T F T <---
159792024-01-10T18:01:40.923Z 6 120-139 147 147 147 265 265 265 T F T <---
159802024-01-10T18:01:40.923Z 7 140-159 140 140 140 252 252 252 T F T <---
159812024-01-10T18:01:40.923Z 8 160-179 147 147 147 265 265 265 T F T <---
159822024-01-10T18:01:40.923Z 9 180-199 147 147 147 265 265 265 F F F
159832024-01-10T18:01:40.923Z 10 200-219 147 147 147 265 265 265 F F F
159842024-01-10T18:01:40.926Z 11 220-239 146 146 146 263 263 263 T F T <---
159852024-01-10T18:01:40.926Z 12 240-259 148 146 148 266 264 266 F F F <---
159862024-01-10T18:01:40.926Z 13 260-279 148 146 148 266 264 266 F F F <---
159872024-01-10T18:01:40.927Z 14 280-299 146 146 146 263 263 263 F F F
159882024-01-10T18:01:40.927Z 15 300-319 142 142 142 254 254 254 F F F
159892024-01-10T18:01:40.927Z 16 320-339 148 140 148 266 252 266 F F F <---
159902024-01-10T18:01:40.927Z 17 340-359 148 144 148 266 258 266 F T F <---
159912024-01-10T18:01:40.927Z 18 360-379 148 144 148 266 258 266 F T F <---
159922024-01-10T18:01:40.927Z 19 380-399 146 146 146 263 263 263 F F F
159932024-01-10T18:01:40.927Z 20 400-419 148 126 148 266 230 266 T F T <---
159942024-01-10T18:01:40.927Z 21 420-439 148 146 148 266 264 266 T F T <---
159952024-01-10T18:01:40.927Z 22 440-459 148 146 148 266 264 266 F F F <---
159962024-01-10T18:01:40.927Z 23 460-479 146 146 146 264 264 264 F F F
159972024-01-10T18:01:40.927Z 24 480-499 147 147 147 265 265 265 F F F
159982024-01-10T18:01:40.927Z 25 500-519 144 144 144 260 260 260 F F F
159992024-01-10T18:01:40.927Z 26 520-539 142 142 142 255 255 255 F F F
160002024-01-10T18:01:40.927Z 27 540-559 148 143 148 266 256 266 F F F <---
160012024-01-10T18:01:40.927Z 28 560-579 144 144 144 258 258 258 F F F
160022024-01-10T18:01:40.927Z 29 580-599 143 143 143 256 256 256 F F F
160032024-01-10T18:01:40.927ZMax gen: 148, Max flush: 266
160042024-01-10T18:01:40.927ZError: Difference in extent metadata found!
160052024-01-10T18:01:40.930ZA Difference in extent metadata is expected here
160062024-01-10T18:01:40.931ZOn loop 74
160072024-01-10T18:01:40.931Z
160082024-01-10T18:01:40.931ZVerifying data now
160092024-01-10T18:01:47.359ZNo change after verify
160102024-01-10T18:01:47.359ZLoop: 74 Downstairs dump after verify (and repair):
160112024-01-10T18:01:47.385ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160122024-01-10T18:01:47.386Z 0 000-019 145 145 145 261 261 261 F F F
160132024-01-10T18:01:47.386Z 1 020-039 149 149 149 267 267 267 F F F
160142024-01-10T18:01:47.386Z 2 040-059 146 146 146 264 264 264 F F F
160152024-01-10T18:01:47.386Z 3 060-079 146 146 146 263 263 263 F F F
160162024-01-10T18:01:47.387Z 4 080-099 146 146 146 263 263 263 F F F
160172024-01-10T18:01:47.387Z 5 100-119 149 149 149 267 267 267 F F F
160182024-01-10T18:01:47.387Z 6 120-139 149 149 149 267 267 267 F F F
160192024-01-10T18:01:47.387Z 7 140-159 149 149 149 267 267 267 F F F
160202024-01-10T18:01:47.387Z 8 160-179 149 149 149 267 267 267 F F F
160212024-01-10T18:01:47.387Z 9 180-199 147 147 147 265 265 265 F F F
160222024-01-10T18:01:47.387Z 10 200-219 147 147 147 265 265 265 F F F
160232024-01-10T18:01:47.387Z 11 220-239 149 149 149 267 267 267 F F F
160242024-01-10T18:01:47.387Z 12 240-259 148 148 148 266 266 266 F F F
160252024-01-10T18:01:47.388Z 13 260-279 148 148 148 266 266 266 F F F
160262024-01-10T18:01:47.472Z 14 280-299 146 146 146 263 263 263 F F F
160272024-01-10T18:01:47.472Z 15 300-319 142 142 142 254 254 254 F F F
160282024-01-10T18:01:47.472Z 16 320-339 148 148 148 266 266 266 F F F
160292024-01-10T18:01:47.472Z 17 340-359 148 148 148 266 266 266 F F F
160302024-01-10T18:01:47.472Z 18 360-379 148 148 148 266 266 266 F F F
160312024-01-10T18:01:47.472Z 19 380-399 146 146 146 263 263 263 F F F
160322024-01-10T18:01:47.472Z 20 400-419 149 149 149 267 267 267 F F F
160332024-01-10T18:01:47.473Z 21 420-439 149 149 149 267 267 267 F F F
160342024-01-10T18:01:47.473Z 22 440-459 148 148 148 266 266 266 F F F
160352024-01-10T18:01:47.473Z 23 460-479 146 146 146 264 264 264 F F F
160362024-01-10T18:01:47.473Z 24 480-499 147 147 147 265 265 265 F F F
160372024-01-10T18:01:47.473Z 25 500-519 144 144 144 260 260 260 F F F
160382024-01-10T18:01:47.473Z 26 520-539 142 142 142 255 255 255 F F F
160392024-01-10T18:01:47.473Z 27 540-559 148 148 148 266 266 266 F F F
160402024-01-10T18:01:47.473Z 28 560-579 144 144 144 258 258 258 F F F
160412024-01-10T18:01:47.473Z 29 580-599 143 143 143 256 256 256 F F F
160422024-01-10T18:01:47.473ZMax gen: 149, Max flush: 267
160432024-01-10T18:01:47.473Z
160442024-01-10T18:01:47.473ZBegin loop 75
160452024-01-10T18:01:47.473ZDownstairs to restart: 1
160462024-01-10T18:01:47.491Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.421907475Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758}
160472024-01-10T18:01:47.492Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.422477289Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758}
160482024-01-10T18:01:47.492Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.422547784Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758}
160492024-01-10T18:01:47.493Z{"msg":"Crucible b8873125-cfd2-4d67-aa0e-b14adec9a64b has session id: 27187a7d-2af3-4909-87e2-2e457bf36829","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.422573434Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160502024-01-10T18:01:47.494Z{"msg":"Upstairs opts: Upstairs UUID: b8873125-cfd2-4d67-aa0e-b14adec9a64b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.422692997Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160512024-01-10T18:01:47.494Z{"msg":"Crucible stats registered with UUID: b8873125-cfd2-4d67-aa0e-b14adec9a64b","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
160522024-01-10T18:01:47.497Z2024-01-10T18:01:47.422811641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160532024-01-10T18:01:47.497Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423002245Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758}
160542024-01-10T18:01:47.498Z{"msg":"b8873125-cfd2-4d67-aa0e-b14adec9a64b active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423076072Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160552024-01-10T18:01:47.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423145602Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160562024-01-10T18:01:47.498Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423188942Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160572024-01-10T18:01:47.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423260762Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160582024-01-10T18:01:47.498Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423306205Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160592024-01-10T18:01:47.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423380051Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160602024-01-10T18:01:47.499Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423425164Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160612024-01-10T18:01:47.499Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423491066Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160622024-01-10T18:01:47.499Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423535761Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160632024-01-10T18:01:47.502Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423605665Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160642024-01-10T18:01:47.502Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423665424Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160652024-01-10T18:01:47.503Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.42373865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160662024-01-10T18:01:47.503Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.423859882Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160672024-01-10T18:01:47.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.424984027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160682024-01-10T18:01:47.503Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.425108688Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160692024-01-10T18:01:47.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.425198302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160702024-01-10T18:01:47.504Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.425240387Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160712024-01-10T18:01:47.504Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.425968009Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160722024-01-10T18:01:47.504Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.426186637Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160732024-01-10T18:01:47.505Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.426406845Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160742024-01-10T18:01:47.506Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.426439453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160752024-01-10T18:01:47.506Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.42670981Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160762024-01-10T18:01:47.506Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.42676457Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160772024-01-10T18:01:47.506Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.426789768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160782024-01-10T18:01:47.506Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.427040047Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160792024-01-10T18:01:47.507Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.427108254Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160802024-01-10T18:01:47.507Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.427174272Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160812024-01-10T18:01:47.507Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.427469371Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160822024-01-10T18:01:47.507Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.42814857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160832024-01-10T18:01:47.507Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428200647Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160842024-01-10T18:01:47.507Z{"msg":"[0]R flush_numbers[0..12]: [261, 267, 264, 263, 263, 267, 267, 267, 267, 265, 265, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428233216Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160852024-01-10T18:01:47.508Z{"msg":"[0]R generation[0..12]: [145, 149, 146, 146, 146, 149, 149, 149, 149, 147, 147, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428298415Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160862024-01-10T18:01:47.508Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428326593Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160872024-01-10T18:01:47.508Z{"msg":"[1]R flush_numbers[0..12]: [261, 267, 264, 263, 263, 267, 267, 267, 267, 265, 265, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428368473Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160882024-01-10T18:01:47.508Z{"msg":"[1]R generation[0..12]: [145, 149, 146, 146, 146, 149, 149, 149, 149, 147, 147, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428393126Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160892024-01-10T18:01:47.508Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428416631Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160902024-01-10T18:01:47.509Z{"msg":"[2]R flush_numbers[0..12]: [261, 267, 264, 263, 263, 267, 267, 267, 267, 265, 265, 267]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
160912024-01-10T18:01:47.509Z,"time":"2024-01-10T18:01:47.428462652Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160922024-01-10T18:01:47.509Z{"msg":"[2]R generation[0..12]: [145, 149, 146, 146, 146, 149, 149, 149, 149, 147, 147, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428489412Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160932024-01-10T18:01:47.510Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428531049Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160942024-01-10T18:01:47.510Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428554482Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160952024-01-10T18:01:47.511Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428576437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160962024-01-10T18:01:47.511Z{"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428598854Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160972024-01-10T18:01:47.511Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428638958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160982024-01-10T18:01:47.511Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428661827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
160992024-01-10T18:01:47.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428682263Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
161002024-01-10T18:01:47.512Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428703524Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
161012024-01-10T18:01:47.512Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428756308Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
161022024-01-10T18:01:47.512Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428779566Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
161032024-01-10T18:01:47.513Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428828496Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
161042024-01-10T18:01:47.513Z{"msg":"b8873125-cfd2-4d67-aa0e-b14adec9a64b is now active with session: 27187a7d-2af3-4909-87e2-2e457bf36829","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428851151Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
161052024-01-10T18:01:47.514Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428870808Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758,"session_id":"27187a7d-2af3-4909-87e2-2e457bf36829"}
161062024-01-10T18:01:47.514Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:47.428923721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1758}
161072024-01-10T18:01:47.514ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161082024-01-10T18:01:47.515ZLoading write count information from file "/tmp/test_repair_verify.data"
161092024-01-10T18:01:47.515ZRun Repair workload
161102024-01-10T18:01:47.515Z01/30 Write block 179 len 32768 data: 10 13 11 11 10 9 11 11
161112024-01-10T18:01:47.515Z02/30 Write block 534 len 16384 data: 15 14 15 13
161122024-01-10T18:01:47.515Z03/30 Read block 283 len 20480
161132024-01-10T18:01:47.515Z04/30 Write block 341 len 16384 data: 12 15 16 15
161142024-01-10T18:01:47.516Z05/30 Read block 241 len 12288
161152024-01-10T18:01:47.516Z06/30 Write block 67 len 4096 data: 12
161162024-01-10T18:01:47.516Z07/30 Read block 277 len 16384
161172024-01-10T18:01:47.516Z08/30 Flush
161182024-01-10T18:01:47.516Z09/30 Write block 305 len 16384 data: 11 8 6 4
161192024-01-10T18:01:47.517Z10/30 Write block 204 len 12288 data: 10 10 10
161202024-01-10T18:01:47.517Z11/30 Read block 461 len 4096
161212024-01-10T18:01:47.517Z12/30 Write block 134 len 20480 data: 8 9 7 10 11
161222024-01-10T18:01:47.518Z13/30 Read block 54 len 32768
161232024-01-10T18:01:47.518Z14/30 Read block 315 len 8192
161242024-01-10T18:01:47.518Z15/30 Write block 71 len 4096 data: 8
161252024-01-10T18:01:47.519Z16/30 Write block 258 len 28672 data: 13 13 12 11 9 9 10
161262024-01-10T18:01:47.519Z17/30 Read block 533 len 36864
161272024-01-10T18:01:47.519Z18/30 Read block 547 len 24576
161282024-01-10T18:01:47.519Z19/30 Read block 62 len 8192
161292024-01-10T18:01:47.520Z20/30 Write block 356 len 12288 data: 9 11 14
161302024-01-10T18:01:47.520Z21/30 Write block 554 len 24576 data: 16 13 12 11 9 9
161312024-01-10T18:01:47.520Z22/30 Flush
161322024-01-10T18:01:47.520Z23/30 Write block 371 len 28672 data: 9 9 10 9 9 7 6
161332024-01-10T18:01:47.520Z24/30 Read block 243 len 40960
161342024-01-10T18:01:47.520Z25/30 Write block 58 len 24576 data: 13 12 12 10 10 10
161352024-01-10T18:01:47.521Z26/30 Write block 517 len 16384 data: 13 12 10 12
161362024-01-10T18:01:47.521Z27/30 Read block 171 len 32768
161372024-01-10T18:01:47.521Z28/30 Write block 16 len 36864 data: 10 10 12 11 10 11 9 11 9
161382024-01-10T18:01:47.521Z29/30 Write block 161 len 32768 data: 14 15 13 15 16 16 16 14
161392024-01-10T18:01:47.521Z30/30 Write block 147 len 12288 data: 12 12 11
161402024-01-10T18:01:47.521Z----------------------------------------------------------------
161412024-01-10T18:01:47.522Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
161422024-01-10T18:01:47.523ZStates: Active Active Active
161432024-01-10T18:01:47.523ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
161442024-01-10T18:01:47.523Z 1 Acked 1000 Write 8 Done Sent Done false
161452024-01-10T18:01:47.523Z 2 Acked 1001 Write 4 Done Sent Done false
161462024-01-10T18:01:47.532Z 3 Acked 1002 Read 5 Done Sent Done false
161472024-01-10T18:01:47.532Z 4 Acked 1003 Write 4 Done Sent Done false
161482024-01-10T18:01:47.532Z 5 Acked 1004 Read 3 Done Sent Done false
161492024-01-10T18:01:47.532Z 6 Acked 1005 Write 1 Done Sent Done false
161502024-01-10T18:01:47.533Z 7 Acked 1006 Read 4 Done Sent Done false
161512024-01-10T18:01:47.533Z 8 Acked 1007 Flush 0 Done Sent Done false
161522024-01-10T18:01:47.533Z 9 Acked 1008 Write 4 Done Sent Done false
161532024-01-10T18:01:47.533Z 10 Acked 1009 Write 3 Done Sent Done false
161542024-01-10T18:01:47.533Z 11 Acked 1010 Read 1 Done Sent Done false
161552024-01-10T18:01:47.533Z 12 Acked 1011 Write 5 Done Sent Done false
161562024-01-10T18:01:47.533Z 13 Acked 1012 Read 8 Done Sent Done false
161572024-01-10T18:01:47.533Z 14 Acked 1013 Read 2 Done Sent Done false
161582024-01-10T18:01:47.533Z 15 Acked 1014 Write 1 Done Sent Done false
161592024-01-10T18:01:47.533Z 16 Acked 1015 Write 7 Done Sent Done false
161602024-01-10T18:01:47.533Z 17 Acked 1016 Read 9 Done Sent Done false
161612024-01-10T18:01:47.533Z 18 Acked 1017 Read 6 Done Sent Done false
161622024-01-10T18:01:47.534Z 19 Acked 1018 Read 2 Done Sent Done false
161632024-01-10T18:01:47.534Z 20 Acked 1019 Write 3 Done Sent Done false
161642024-01-10T18:01:47.534Z 21 Acked 1020 Write 6 Done Sent Done false
161652024-01-10T18:01:47.534Z 22 Acked 1021 Flush 0 Done Sent Done false
161662024-01-10T18:01:47.534Z 23 Acked 1022 Write 7 Done Sent Done false
161672024-01-10T18:01:47.534Z 24 Acked 1023 Read 10 Done Sent Done false
161682024-01-10T18:01:47.534Z 25 Acked 1024 Write 6 Done Sent Done false
161692024-01-10T18:01:47.534Z 26 Acked 1025 Write 4 Done Sent Done false
161702024-01-10T18:01:47.534Z 27 Acked 1026 Read 8 Done Sent Done false
161712024-01-10T18:01:47.534Z 28 Acked 1027 Write 9 Done Sent Done false
161722024-01-10T18:01:47.534Z 29 Acked 1028 Write 8 Sent Sent Sent false
161732024-01-10T18:01:47.534Z 30 Acked 1029 Write 3 Sent Sent Sent false
161742024-01-10T18:01:47.535Z STATES DS:0 DS:1 DS:2 TOTAL
161752024-01-10T18:01:47.535Z New 0 0 0 0
161762024-01-10T18:01:47.535Z Sent 2 30 2 34
161772024-01-10T18:01:47.535Z Done 28 0 28 56
161782024-01-10T18:01:47.535Z Skipped 0 0 0 0
161792024-01-10T18:01:47.535Z Error 0 0 0 0
161802024-01-10T18:01:47.535ZLast Flush: 1021 0 1021
161812024-01-10T18:01:47.535ZDownstairs last five completed:
161822024-01-10T18:01:47.535ZUpstairs last five completed: 30 29 28 27 26
161832024-01-10T18:01:47.535ZWrote out file "/tmp/test_repair_verify.data"
161842024-01-10T18:01:47.535Z
161852024-01-10T18:01:47.536ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
161862024-01-10T18:01:47.536ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161872024-01-10T18:01:47.536Z 0 000-019 145 145 145 261 261 261 T F T <---
161882024-01-10T18:01:47.536Z 1 020-039 149 149 149 267 267 267 T F T <---
161892024-01-10T18:01:47.536Z 2 040-059 146 146 146 264 264 264 T F T <---
161902024-01-10T18:01:47.536Z 3 060-079 150 146 150 269 263 269 T F T <---
161912024-01-10T18:01:47.536Z 4 080-099 146 146 146 263 263 263 F F F
161922024-01-10T18:01:47.536Z 5 100-119 149 149 149 267 267 267 F F F
161932024-01-10T18:01:47.536Z 6 120-139 150 149 150 269 267 269 F F F <---
161942024-01-10T18:01:47.537Z 7 140-159 149 149 149 267 267 267 T F T <---
161952024-01-10T18:01:47.537Z 8 160-179 150 149 150 268 267 268 T F T <---
161962024-01-10T18:01:47.538Z 9 180-199 150 147 150 268 265 268 F F F <---
161972024-01-10T18:01:47.538Z 10 200-219 150 147 150 269 265 269 F F F <---
161982024-01-10T18:01:47.538Z 11 220-239 149 149 149 267 267 267 F F F
161992024-01-10T18:01:47.538Z 12 240-259 150 148 150 269 266 269 F F F <---
162002024-01-10T18:01:47.538Z 13 260-279 150 148 150 269 266 269 F F F <---
162012024-01-10T18:01:47.538Z 14 280-299 146 146 146 263 263 263 F F F
162022024-01-10T18:01:47.538Z 15 300-319 150 142 150 269 254 269 F F F <---
162032024-01-10T18:01:47.538Z 16 320-339 148 148 148 266 266 266 F F F
162042024-01-10T18:01:47.538Z 17 340-359 150 148 150 269 266 269 F F F <---
162052024-01-10T18:01:47.538Z 18 360-379 148 148 148 266 266 266 T F T <---
162062024-01-10T18:01:47.538Z 19 380-399 146 146 146 263 263 263 F F F
162072024-01-10T18:01:47.538Z 20 400-419 149 149 149 267 267 267 F F F
162082024-01-10T18:01:47.539Z 21 420-439 149 149 149 267 267 267 F F F
162092024-01-10T18:01:47.539Z 22 440-459 148 148 148 266 266 266 F F F
162102024-01-10T18:01:47.539Z 23 460-479 146 146 146 264 264 264 F F F
162112024-01-10T18:01:47.539Z 24 480-499 147 147 147 265 265 265 F F F
162122024-01-10T18:01:47.539Z 25 500-519 144 144 144 260 260 260 T F T <---
162132024-01-10T18:01:47.539Z 26 520-539 150 142 150 268 255 268 T F T <---
162142024-01-10T18:01:47.539Z 27 540-559 150 148 150 269 266 269 F F F <---
162152024-01-10T18:01:47.539Z 28 560-579 144 144 144 258 258 258 F F F
162162024-01-10T18:01:47.539Z 29 580-599 143 143 143 256 256 256 F F F
162172024-01-10T18:01:47.539ZMax gen: 150, Max flush: 269
162182024-01-10T18:01:47.540ZError: Difference in extent metadata found!
162192024-01-10T18:01:47.540ZA Difference in extent metadata is expected here
162202024-01-10T18:01:47.540ZOn loop 75
162212024-01-10T18:01:47.541Z
162222024-01-10T18:01:47.541ZVerifying data now
162232024-01-10T18:01:53.132ZNo change after verify
162242024-01-10T18:01:53.132ZLoop: 75 Downstairs dump after verify (and repair):
162252024-01-10T18:01:53.155ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162262024-01-10T18:01:53.155Z 0 000-019 151 151 151 270 270 270 F F F
162272024-01-10T18:01:53.155Z 1 020-039 151 151 151 270 270 270 F F F
162282024-01-10T18:01:53.155Z 2 040-059 151 151 151 270 270 270 F F F
162292024-01-10T18:01:53.155Z 3 060-079 151 151 151 270 270 270 F F F
162302024-01-10T18:01:53.155Z 4 080-099 146 146 146 263 263 263 F F F
162312024-01-10T18:01:53.155Z 5 100-119 149 149 149 267 267 267 F F F
162322024-01-10T18:01:53.155Z 6 120-139 150 150 150 269 269 269 F F F
162332024-01-10T18:01:53.155Z 7 140-159 151 151 151 270 270 270 F F F
162342024-01-10T18:01:53.155Z 8 160-179 151 151 151 270 270 270 F F F
162352024-01-10T18:01:53.156Z 9 180-199 150 150 150 268 268 268 F F F
162362024-01-10T18:01:53.156Z 10 200-219 150 150 150 269 269 269 F F F
162372024-01-10T18:01:53.156Z 11 220-239 149 149 149 267 267 267 F F F
162382024-01-10T18:01:53.156Z 12 240-259 150 150 150 269 269 269 F F F
162392024-01-10T18:01:53.156Z 13 260-279 150 150 150 269 269 269 F F F
162402024-01-10T18:01:53.156Z 14 280-299 146 146 146 263 263 263 F F F
162412024-01-10T18:01:53.156Z 15 300-319 150 150 150 269 269 269 F F F
162422024-01-10T18:01:53.158Z 16 320-339 148 148 148 266 266 266 F F F
162432024-01-10T18:01:53.158Z 17 340-359 150 150 150 269 269 269 F F F
162442024-01-10T18:01:53.158Z 18 360-379 151 151 151 270 270 270 F F F
162452024-01-10T18:01:53.158Z 19 380-399 146 146 146 263 263 263 F F F
162462024-01-10T18:01:53.158Z 20 400-419 149 149 149 267 267 267 F F F
162472024-01-10T18:01:53.161Z 21 420-439 149 149 149 267 267 267 F F F
162482024-01-10T18:01:53.162Z 22 440-459 148 148 148 266 266 266 F F F
162492024-01-10T18:01:53.162Z 23 460-479 146 146 146 264 264 264 F F F
162502024-01-10T18:01:53.162Z 24 480-499 147 147 147 265 265 265 F F F
162512024-01-10T18:01:53.162Z 25 500-519 151 151 151 270 270 270 F F F
162522024-01-10T18:01:53.162Z 26 520-539 151 151 151 270 270 270 F F F
162532024-01-10T18:01:53.162Z 27 540-559 150 150 150 269 269 269 F F F
162542024-01-10T18:01:53.162Z 28 560-579 144 144 144 258 258 258 F F F
162552024-01-10T18:01:53.162Z 29 580-599 143 143 143 256 256 256 F F F
162562024-01-10T18:01:53.162ZMax gen: 151, Max flush: 270
162572024-01-10T18:01:53.162Z
162582024-01-10T18:01:53.162ZBegin loop 76
162592024-01-10T18:01:53.162ZDownstairs to restart: 1
162602024-01-10T18:01:53.195Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.199821463Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766}
162612024-01-10T18:01:53.195Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.200373449Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766}
162622024-01-10T18:01:53.195Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.200432054Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766}
162632024-01-10T18:01:53.200Z{"msg":"Crucible b4ddef38-c9b3-4f30-87a3-4ea972aa9242 has session id: bca1fb52-ddfd-42db-a498-29a7bb1d2637","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.200474205Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162642024-01-10T18:01:53.200Z{"msg":"Upstairs opts: Upstairs UUID: b4ddef38-c9b3-4f30-87a3-4ea972aa9242, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.200539265Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162652024-01-10T18:01:53.200Z{"msg":"Crucible stats registered with UUID: b4ddef38-c9b3-4f30-87a3-4ea972aa9242","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.20057835Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162662024-01-10T18:01:53.200ZCrucible runtime is spawned
162672024-01-10T18:01:53.200Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.201198204Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766}
162682024-01-10T18:01:53.200Z{"msg":"b4ddef38-c9b3-4f30-87a3-4ea972aa9242 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.201452459Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162692024-01-10T18:01:53.200Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.201518275Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162702024-01-10T18:01:53.200Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.201662861Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162712024-01-10T18:01:53.200Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.201688068Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162722024-01-10T18:01:53.200Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.201802976Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162732024-01-10T18:01:53.200Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.201914184Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162742024-01-10T18:01:53.201Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.201994548Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162752024-01-10T18:01:53.201Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.202039157Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162762024-01-10T18:01:53.201Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.202358083Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162772024-01-10T18:01:53.201Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.202402892Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162782024-01-10T18:01:53.201Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.202440695Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162792024-01-10T18:01:53.201Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.202505298Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162802024-01-10T18:01:53.201Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.202575943Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162812024-01-10T18:01:53.201Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.203488427Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162822024-01-10T18:01:53.201Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.203660547Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162832024-01-10T18:01:53.201Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.20371068Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162842024-01-10T18:01:53.202Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.203853732Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162852024-01-10T18:01:53.202Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.204549093Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162862024-01-10T18:01:53.202Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.204692176Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162872024-01-10T18:01:53.207Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.205139669Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162882024-01-10T18:01:53.207Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.205257319Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162892024-01-10T18:01:53.207Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.205657414Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162902024-01-10T18:01:53.207Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.205719233Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162912024-01-10T18:01:53.207Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.205744831Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162922024-01-10T18:01:53.208Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.205850878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162932024-01-10T18:01:53.208Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.205906152Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162942024-01-10T18:01:53.208Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.205931717Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162952024-01-10T18:01:53.209Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.206303223Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162962024-01-10T18:01:53.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207009139Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162972024-01-10T18:01:53.209Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207071091Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162982024-01-10T18:01:53.209Z{"msg":"[0]R flush_numbers[0..12]: [270, 270, 270, 270, 263, 267, 269, 270, 270, 268, 269, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207097359Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
162992024-01-10T18:01:53.209Z{"msg":"[0]R generation[0..12]: [151, 151, 151, 151, 146, 149, 150, 151, 151, 150, 150, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207121016Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163002024-01-10T18:01:53.209Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207152152Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163012024-01-10T18:01:53.209Z{"msg":"[1]R flush_numbers[0..12]: [270, 270, 270, 270, 263, 267, 269, 270, 270, 268, 269, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207198187Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163022024-01-10T18:01:53.209Z{"msg":"[1]R generation[0..12]: [151, 151, 151, 151, 146, 149, 150, 151, 151, 150, 150, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207224233Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163032024-01-10T18:01:53.209Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207247869Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163042024-01-10T18:01:53.209Z{"msg":"[2]R flush_numbers[0..12]: [270, 270, 270, 270, 263, 267, 269, 270, 270, 268, 269, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.20728971Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163052024-01-10T18:01:53.209Z{"msg":"[2]R generation[0..12]: [151, 151, 151, 151, 146, 149, 150, 151, 151, 150, 150, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207314378Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163062024-01-10T18:01:53.209Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207351847Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
163072024-01-10T18:01:53.209Z,"":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163082024-01-10T18:01:53.209Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207424315Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163092024-01-10T18:01:53.209Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207454024Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163102024-01-10T18:01:53.209Z{"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207508833Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163112024-01-10T18:01:53.209Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207552995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163122024-01-10T18:01:53.209Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207619903Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163132024-01-10T18:01:53.210Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207645603Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163142024-01-10T18:01:53.210Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207693403Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163152024-01-10T18:01:53.210Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207719826Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163162024-01-10T18:01:53.210Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207749549Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163172024-01-10T18:01:53.210Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.2078179Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163182024-01-10T18:01:53.210Z{"msg":"b4ddef38-c9b3-4f30-87a3-4ea972aa9242 is now active with session: bca1fb52-ddfd-42db-a498-29a7bb1d2637","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207855831Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163192024-01-10T18:01:53.210Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207877138Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766,"session_id":"bca1fb52-ddfd-42db-a498-29a7bb1d2637"}
163202024-01-10T18:01:53.210Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:53.207918002Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1766}
163212024-01-10T18:01:53.210ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163222024-01-10T18:01:53.210ZLoading write count information from file "/tmp/test_repair_verify.data"
163232024-01-10T18:01:53.210ZRun Repair workload
163242024-01-10T18:01:53.210Z01/30 Write block 151 len 40960 data: 10 9 10 11 12 14 12 10 9 10
163252024-01-10T18:01:53.214Z02/30 Flush
163262024-01-10T18:01:53.214Z03/30 Write block 25 len 8192 data: 10 10
163272024-01-10T18:01:53.215Z04/30 Read block 319 len 20480
163282024-01-10T18:01:53.216Z05/30 Read block 322 len 40960
163292024-01-10T18:01:53.220Z06/30 Read block 28 len 20480
163302024-01-10T18:01:53.220Z07/30 Read block 136 len 24576
163312024-01-10T18:01:53.226Z08/30 Write block 473 len 12288 data: 6 8 7
163322024-01-10T18:01:53.226Z09/30 Read block 175 len 36864
163332024-01-10T18:01:53.239Z10/30 Read block 6 len 20480
163342024-01-10T18:01:53.239Z11/30 Write block 64 len 16384 data: 11 13 15 13
163352024-01-10T18:01:53.239Z12/30 Read block 445 len 24576
163362024-01-10T18:01:53.239Z13/30 Read block 519 len 8192
163372024-01-10T18:01:53.239Z14/30 Write block 145 len 12288 data: 11 11 13
163382024-01-10T18:01:53.249Z15/30 Write block 580 len 32768 data: 13 12 12 13 10 9 11 10
163392024-01-10T18:01:53.249Z16/30 Read block 521 len 8192
163402024-01-10T18:01:53.249Z17/30 Write block 522 len 20480 data: 16 11 9 11 13
163412024-01-10T18:01:53.249Z18/30 Read block 463 len 32768
163422024-01-10T18:01:53.250Z19/30 Read block 569 len 28672
163432024-01-10T18:01:53.250Z20/30 Write block 160 len 4096 data: 11
163442024-01-10T18:01:53.250Z21/30 Read block 22 len 24576
163452024-01-10T18:01:53.250Z22/30 Read block 311 len 8192
163462024-01-10T18:01:53.252Z23/30 Read block 19 len 12288
163472024-01-10T18:01:53.253Z24/30 Read block 444 len 36864
163482024-01-10T18:01:53.253Z25/30 Read block 571 len 16384
163492024-01-10T18:01:53.256Z26/30 Write block 434 len 32768 data: 10 11 11 12 12 12 10 13
163502024-01-10T18:01:53.256Z27/30 Read block 214 len 16384
163512024-01-10T18:01:53.261Z28/30 Write block 401 len 36864 data: 10 10 13 14 16 17 17 14 11
163522024-01-10T18:01:53.261Z29/30 Write block 349 len 16384 data: 9 8 9 11
163532024-01-10T18:01:53.262Z30/30 Read block 452 len 28672
163542024-01-10T18:01:53.266Z----------------------------------------------------------------
163552024-01-10T18:01:53.266Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:28
163562024-01-10T18:01:53.266ZStates: Active Active Active
163572024-01-10T18:01:53.266ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
163582024-01-10T18:01:53.266Z 3 Acked 1002 Write 2 Done Sent Done false
163592024-01-10T18:01:53.266Z 4 Acked 1003 Read 5 Done Sent Done false
163602024-01-10T18:01:53.266Z 5 Acked 1004 Read 10 Done Sent Done false
163612024-01-10T18:01:53.266Z 6 Acked 1005 Read 5 Done Sent Done false
163622024-01-10T18:01:53.266Z 7 Acked 1006 Read 6 Done Sent Done false
163632024-01-10T18:01:53.266Z 8 Acked 1007 Write 3 Done Sent Done false
163642024-01-10T18:01:53.266Z 9 Acked 1008 Read 9 Done Sent Done false
163652024-01-10T18:01:53.267Z 10 Acked 1009 Read 5 Done Sent Done false
163662024-01-10T18:01:53.267Z 11 Acked 1010 Write 4 Done Sent Done false
163672024-01-10T18:01:53.267Z 12 Acked 1011 Read 6 Done Sent Done false
163682024-01-10T18:01:53.267Z 13 Acked 1012 Read 2 Done Sent Done false
163692024-01-10T18:01:53.267Z 14 Acked 1013 Write 3 Done Sent Done false
163702024-01-10T18:01:53.267Z 15 Acked 1014 Write 8 Done Sent Done false
163712024-01-10T18:01:53.267Z 16 Acked 1015 Read 2 Done Sent Done false
163722024-01-10T18:01:53.267Z 17 Acked 1016 Write 5 Done Sent Done false
163732024-01-10T18:01:53.267Z 18 Acked 1017 Read 8 Done Sent Done false
163742024-01-10T18:01:53.267Z 19 Acked 1018 Read 7 Done Sent Done false
163752024-01-10T18:01:53.267Z 20 Acked 1019 Write 1 Done Sent Done false
163762024-01-10T18:01:53.268Z 21 Acked 1020 Read 6 Done Sent Done false
163772024-01-10T18:01:53.268Z 22 Acked 1021 Read 2 Done Sent Done false
163782024-01-10T18:01:53.268Z 23 Acked 1022 Read 3 Done Sent Done false
163792024-01-10T18:01:53.268Z 24 Acked 1023 Read 9 Done Sent Done false
163802024-01-10T18:01:53.268Z 25 Acked 1024 Read 4 Done Sent Done false
163812024-01-10T18:01:53.268Z 26 Acked 1025 Write 8 Done Sent Done false
163822024-01-10T18:01:53.268Z 27 Acked 1026 Read 4 Done Sent Done false
163832024-01-10T18:01:53.268Z 28 Acked 1027 Write 9 Done Sent Done false
163842024-01-10T18:01:53.268Z 29 Acked 1028 Write 4 Done Sent Done false
163852024-01-10T18:01:53.268Z 30 Acked 1029 Read 7 Done Sent Done false
163862024-01-10T18:01:53.268Z STATES DS:0 DS:1 DS:2 TOTAL
163872024-01-10T18:01:53.268Z New 0 0 0 0
163882024-01-10T18:01:53.268Z Sent 0 28 0 28
163892024-01-10T18:01:53.268Z Done 28 0 28 56
163902024-01-10T18:01:53.268Z Skipped 0 0 0 0
163912024-01-10T18:01:53.268Z Error 0 0 0 0
163922024-01-10T18:01:53.268ZLast Flush: 1001 1001 1001
163932024-01-10T18:01:53.268ZDownstairs last five completed: 1001 1000
163942024-01-10T18:01:53.268ZUpstairs last five completed: 30 29 28 27 26
163952024-01-10T18:01:53.268ZWrote out file "/tmp/test_repair_verify.data"
163962024-01-10T18:01:53.272Z
163972024-01-10T18:01:53.276ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
163982024-01-10T18:01:53.305ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163992024-01-10T18:01:53.305Z 0 000-019 151 151 151 270 270 270 F F F
164002024-01-10T18:01:53.305Z 1 020-039 151 151 151 270 270 270 T F T <---
164012024-01-10T18:01:53.305Z 2 040-059 151 151 151 270 270 270 F F F
164022024-01-10T18:01:53.305Z 3 060-079 151 151 151 270 270 270 T F T <---
164032024-01-10T18:01:53.305Z 4 080-099 146 146 146 263 263 263 F F F
164042024-01-10T18:01:53.305Z 5 100-119 149 149 149 267 267 267 F F F
164052024-01-10T18:01:53.305Z 6 120-139 150 150 150 269 269 269 F F F
164062024-01-10T18:01:53.305Z 7 140-159 152 152 152 271 271 271 T F T <---
164072024-01-10T18:01:53.305Z 8 160-179 152 152 152 271 271 271 T F T <---
164082024-01-10T18:01:53.305Z 9 180-199 150 150 150 268 268 268 F F F
164092024-01-10T18:01:53.305Z 10 200-219 150 150 150 269 269 269 F F F
164102024-01-10T18:01:53.305Z 11 220-239 149 149 149 267 267 267 F F F
164112024-01-10T18:01:53.305Z 12 240-259 150 150 150 269 269 269 F F F
164122024-01-10T18:01:53.309Z 13 260-279 150 150 150 269 269 269 F F F
164132024-01-10T18:01:53.309Z 14 280-299 146 146 146 263 263 263 F F F
164142024-01-10T18:01:53.309Z 15 300-319 150 150 150 269 269 269 F F F
164152024-01-10T18:01:53.309Z 16 320-339 148 148 148 266 266 266 F F F
164162024-01-10T18:01:53.309Z 17 340-359 150 150 150 269 269 269 T F T <---
164172024-01-10T18:01:53.309Z 18 360-379 151 151 151 270 270 270 F F F
164182024-01-10T18:01:53.309Z 19 380-399 146 146 146 263 263 263 F F F
164192024-01-10T18:01:53.309Z 20 400-419 149 149 149 267 267 267 T F T <---
164202024-01-10T18:01:53.309Z 21 420-439 149 149 149 267 267 267 T F T <---
164212024-01-10T18:01:53.310Z 22 440-459 148 148 148 266 266 266 T F T <---
164222024-01-10T18:01:53.310Z 23 460-479 146 146 146 264 264 264 T F T <---
164232024-01-10T18:01:53.310Z 24 480-499 147 147 147 265 265 265 F F F
164242024-01-10T18:01:53.310Z 25 500-519 151 151 151 270 270 270 F F F
164252024-01-10T18:01:53.310Z 26 520-539 151 151 151 270 270 270 T F T <---
164262024-01-10T18:01:53.310Z 27 540-559 150 150 150 269 269 269 F F F
164272024-01-10T18:01:53.310Z 28 560-579 144 144 144 258 258 258 F F F
164282024-01-10T18:01:53.310Z 29 580-599 143 143 143 256 256 256 T F T <---
164292024-01-10T18:01:53.310ZMax gen: 152, Max flush: 271
164302024-01-10T18:01:53.310ZError: Difference in extent metadata found!
164312024-01-10T18:01:53.314ZA Difference in extent metadata is expected here
164322024-01-10T18:01:53.314ZOn loop 76
164332024-01-10T18:01:53.314Z
164342024-01-10T18:01:53.317ZVerifying data now
164352024-01-10T18:01:58.524ZNo change after verify
164362024-01-10T18:01:58.525ZLoop: 76 Downstairs dump after verify (and repair):
164372024-01-10T18:01:58.549ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164382024-01-10T18:01:58.549Z 0 000-019 151 151 151 270 270 270 F F F
164392024-01-10T18:01:58.549Z 1 020-039 153 153 153 272 272 272 F F F
164402024-01-10T18:01:58.549Z 2 040-059 151 151 151 270 270 270 F F F
164412024-01-10T18:01:58.549Z 3 060-079 153 153 153 272 272 272 F F F
164422024-01-10T18:01:58.549Z 4 080-099 146 146 146 263 263 263 F F F
164432024-01-10T18:01:58.560Z 5 100-119 149 149 149 267 267 267 F F F
164442024-01-10T18:01:58.560Z 6 120-139 150 150 150 269 269 269 F F F
164452024-01-10T18:01:58.560Z 7 140-159 153 153 153 272 272 272 F F F
164462024-01-10T18:01:58.560Z 8 160-179 153 153 153 272 272 272 F F F
164472024-01-10T18:01:58.560Z 9 180-199 150 150 150 268 268 268 F F F
164482024-01-10T18:01:58.560Z 10 200-219 150 150 150 269 269 269 F F F
164492024-01-10T18:01:58.560Z 11 220-239 149 149 149 267 267 267 F F F
164502024-01-10T18:01:58.560Z 12 240-259 150 150 150 269 269 269 F F F
164512024-01-10T18:01:58.561Z 13 260-279 150 150 150 269 269 269 F F F
164522024-01-10T18:01:58.561Z 14 280-299 146 146 146 263 263 263 F F F
164532024-01-10T18:01:58.561Z 15 300-319 150 150 150 269 269 269 F F F
164542024-01-10T18:01:58.561Z 16 320-339 148 148 148 266 266 266 F F F
164552024-01-10T18:01:58.561Z 17 340-359 153 153 153 272 272 272 F F F
164562024-01-10T18:01:58.561Z 18 360-379 151 151 151 270 270 270 F F F
164572024-01-10T18:01:58.561Z 19 380-399 146 146 146 263 263 263 F F F
164582024-01-10T18:01:58.561Z 20 400-419 153 153 153 272 272 272 F F F
164592024-01-10T18:01:58.561Z 21 420-439 153 153 153 272 272 272 F F F
164602024-01-10T18:01:58.561Z 22 440-459 153 153 153 272 272 272 F F F
164612024-01-10T18:01:58.561Z 23 460-479 153 153 153 272 272 272 F F F
164622024-01-10T18:01:58.561Z 24 480-499 147 147 147 265 265 265 F F F
164632024-01-10T18:01:58.561Z 25 500-519 151 151 151 270 270 270 F F F
164642024-01-10T18:01:58.561Z 26 520-539 153 153 153 272 272 272 F F F
164652024-01-10T18:01:58.561Z 27 540-559 150 150 150 269 269 269 F F F
164662024-01-10T18:01:58.561Z 28 560-579 144 144 144 258 258 258 F F F
164672024-01-10T18:01:58.561Z 29 580-599 153 153 153 272 272 272 F F F
164682024-01-10T18:01:58.562ZMax gen: 153, Max flush: 272
164692024-01-10T18:01:58.562Z
164702024-01-10T18:01:58.562ZBegin loop 77
164712024-01-10T18:01:58.562ZDownstairs to restart: 2
164722024-01-10T18:01:58.992Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.581399873Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774}
164732024-01-10T18:01:58.992Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582011848Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774}
164742024-01-10T18:01:58.992Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.58207468Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774}
164752024-01-10T18:01:58.992Z{"msg":"Crucible 7994ce41-a62e-48c3-89a3-e78ab14a753e has session id: 0eeba67f-5e6a-49d6-856f-71ae63cf5b48","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582114571Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164762024-01-10T18:01:58.992Z{"msg":"Upstairs opts: Upstairs UUID: 7994ce41-a62e-48c3-89a3-e78ab14a753e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582168811Z"Crucible runtime is spawned
164772024-01-10T18:01:58.992Z,"hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164782024-01-10T18:01:58.993Z{"msg":"Crucible stats registered with UUID: 7994ce41-a62e-48c3-89a3-e78ab14a753e","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582402894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164792024-01-10T18:01:58.993Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582439123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774}
164802024-01-10T18:01:58.993Z{"msg":"7994ce41-a62e-48c3-89a3-e78ab14a753e active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582572464Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164812024-01-10T18:01:58.993Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582624837Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164822024-01-10T18:01:58.993Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582655078Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164832024-01-10T18:01:58.993Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.5826919Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164842024-01-10T18:01:58.993Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.58272288Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164852024-01-10T18:01:58.993Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582794921Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164862024-01-10T18:01:58.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582839218Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164872024-01-10T18:01:58.994Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.582905399Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164882024-01-10T18:01:58.994Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.583220671Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164892024-01-10T18:01:58.995Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.583345199Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164902024-01-10T18:01:58.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.583391841Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164912024-01-10T18:01:58.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.583489376Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164922024-01-10T18:01:58.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.583570437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164932024-01-10T18:01:58.995Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.584448498Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164942024-01-10T18:01:58.995Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.584591013Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164952024-01-10T18:01:58.995Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.584633701Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164962024-01-10T18:01:58.995Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.584787536Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164972024-01-10T18:01:58.996Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.58485736Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164982024-01-10T18:01:58.996Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.584884716Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
164992024-01-10T18:01:58.996Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.585263729Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165002024-01-10T18:01:58.996Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.585579251Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165012024-01-10T18:01:58.996Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.585987464Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165022024-01-10T18:01:58.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.586269318Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165032024-01-10T18:01:58.996Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.58635247Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165042024-01-10T18:01:58.996Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.586382661Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165052024-01-10T18:01:58.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.586493965Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165062024-01-10T18:01:58.997Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.58654528Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165072024-01-10T18:01:58.997Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.5865703Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165082024-01-10T18:01:58.998Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.586893893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165092024-01-10T18:01:58.998Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.586946003Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165102024-01-10T18:01:58.998Z{"msg":"[0]R flush_numbers[0..12]: [270, 272, 270, 272, 263, 267, 269, 272, 272, 268, 269, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.58697197Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165112024-01-10T18:01:58.998Z{"msg":"[0]R generation[0..12]: [151, 153, 151, 153, 146, 149, 150, 153, 153, 150, 150, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.586995767Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165122024-01-10T18:01:58.998Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587056017Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165132024-01-10T18:01:58.998Z{"msg":"[1]R flush_numbers[0..12]: [270, 272, 270, 272, 263, 267, 269, 272, 272, 268, 269, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587082508Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165142024-01-10T18:01:58.999Z{"msg":"[1]R generation[0..12]: [151, 153, 151, 153, 146, 149, 150, 153, 153, 150, 150, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587124769Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165152024-01-10T18:01:58.999Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587150558Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165162024-01-10T18:01:59.001Z{"msg":"[2]R flush_numbers[0..12]: [270, 272, 270, 272, 263, 267, 269, 272, 272, 268, 269, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587199559Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165172024-01-10T18:01:59.001Z{"msg":"[2]R generation[0..12]: [151, 153, 151, 153, 146, 149, 150, 153, 153, 150, 150, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587229557Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165182024-01-10T18:01:59.002Z{Wait for a query_work_queue command to finish before sending IO
165192024-01-10T18:01:59.002Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587301933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165202024-01-10T18:01:59.002Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587380209Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165212024-01-10T18:01:59.002Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587455846Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165222024-01-10T18:01:59.002Z{"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.58749658Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165232024-01-10T18:01:59.002Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587559413Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165242024-01-10T18:01:59.002Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587600577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165252024-01-10T18:01:59.003Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587667379Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165262024-01-10T18:01:59.003Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.58773096Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165272024-01-10T18:01:59.003Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.58775515Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165282024-01-10T18:01:59.004Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587776608Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165292024-01-10T18:01:59.004Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587813514Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165302024-01-10T18:01:59.004Z{"msg":"7994ce41-a62e-48c3-89a3-e78ab14a753e is now active with session: 0eeba67f-5e6a-49d6-856f-71ae63cf5b48","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.58785074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165312024-01-10T18:01:59.004Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587872409Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774,"session_id":"0eeba67f-5e6a-49d6-856f-71ae63cf5b48"}
165322024-01-10T18:01:59.004Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:01:58.587894976Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1774}
165332024-01-10T18:01:59.004ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165342024-01-10T18:01:59.004ZLoading write count information from file "/tmp/test_repair_verify.data"
165352024-01-10T18:01:59.004ZRun Repair workload
165362024-01-10T18:01:59.004Z01/30 Write block 249 len 20480 data: 11 9 8 9 9
165372024-01-10T18:01:59.004Z02/30 Write block 140 len 12288 data: 13 14 12
165382024-01-10T18:01:59.004Z03/30 Write block 431 len 40960 data: 9 9 11 11 12 12 13 13 13 11
165392024-01-10T18:01:59.005Z04/30 Write block 315 len 28672 data: 12 11 8 9 9 9 11
165402024-01-10T18:01:59.005Z05/30 Write block 67 len 8192 data: 14 12
165412024-01-10T18:01:59.005Z06/30 Write block 538 len 16384 data: 12 12 15 16
165422024-01-10T18:01:59.005Z07/30 Read block 131 len 32768
165432024-01-10T18:01:59.005Z08/30 Write block 128 len 8192 data: 5 5
165442024-01-10T18:01:59.005Z09/30 Write block 572 len 36864 data: 8 9 13 13 14 12 10 11 14
165452024-01-10T18:01:59.005Z10/30 Read block 346 len 40960
165462024-01-10T18:01:59.005Z11/30 Write block 537 len 20480 data: 14 13 13 16 17
165472024-01-10T18:01:59.005Z12/30 Write block 586 len 8192 data: 12 11
165482024-01-10T18:01:59.005Z13/30 Read block 376 len 28672
165492024-01-10T18:01:59.006Z14/30 Read block 348 len 36864
165502024-01-10T18:01:59.006Z15/30 Read block 112 len 24576
165512024-01-10T18:01:59.006Z16/30 Write block 427 len 28672 data: 11 12 11 9 10 10 12
165522024-01-10T18:01:59.006Z17/30 Read block 58 len 12288
165532024-01-10T18:01:59.006Z18/30 Read block 31 len 4096
165542024-01-10T18:01:59.006Z19/30 Read block 479 len 12288
165552024-01-10T18:01:59.006Z20/30 Write block 578 len 16384 data: 11 12 15 13
165562024-01-10T18:01:59.006Z21/30 Read block 77 len 20480
165572024-01-10T18:01:59.006Z22/30 Write block 165 len 28672 data: 17 17 17 15 14 12 9
165582024-01-10T18:01:59.006Z23/30 Write block 468 len 24576 data: 13 9 9 6 5 7
165592024-01-10T18:01:59.006Z24/30 Read block 428 len 24576
165602024-01-10T18:01:59.006Z25/30 Flush
165612024-01-10T18:01:59.007Z26/30 Flush
165622024-01-10T18:01:59.007Z27/30 Write block 323 len 36864 data: 11 13 13 11 10 10 8 7 8
165632024-01-10T18:01:59.007Z28/30 Write block 553 len 16384 data: 17 17 14 13
165642024-01-10T18:01:59.007Z29/30 Write block 58 len 40960 data: 14 13 13 11 11 11 12 14 16 15
165652024-01-10T18:01:59.008Z30/30 Write block 541 len 4096 data: 18
165662024-01-10T18:01:59.008Z----------------------------------------------------------------
165672024-01-10T18:01:59.008Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
165682024-01-10T18:01:59.008ZStates: Active Active Active
165692024-01-10T18:01:59.008ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
165702024-01-10T18:01:59.008Z 1 Acked 1000 Write 5 Done Done Done false
165712024-01-10T18:01:59.008Z 2 Acked 1001 Write 3 Done Done Done false
165722024-01-10T18:01:59.023Z 3 Acked 1002 Write 10 Done Done Done false
165732024-01-10T18:01:59.023Z 4 Acked 1003 Write 7 Done Done Done false
165742024-01-10T18:01:59.024Z 5 Acked 1004 Write 2 Done Done Sent false
165752024-01-10T18:01:59.024Z 6 Acked 1005 Write 4 Done Done Sent false
165762024-01-10T18:01:59.024Z 7 Acked 1006 Read 8 Done Done Sent false
165772024-01-10T18:01:59.024Z 8 Acked 1007 Write 2 Done Done Sent false
165782024-01-10T18:01:59.024Z 9 Acked 1008 Write 9 Done Done Sent false
165792024-01-10T18:01:59.024Z 10 Acked 1009 Read 10 Done Done Sent false
165802024-01-10T18:01:59.024Z 11 Acked 1010 Write 5 Done Done Sent false
165812024-01-10T18:01:59.025Z 12 Acked 1011 Write 2 Done Done Sent false
165822024-01-10T18:01:59.025Z 13 Acked 1012 Read 7 Done Done Sent false
165832024-01-10T18:01:59.025Z 14 Acked 1013 Read 9 Done Done Sent false
165842024-01-10T18:01:59.025Z 15 Acked 1014 Read 6 Done Done Sent false
165852024-01-10T18:01:59.025Z 16 Acked 1015 Write 7 Done Done Sent false
165862024-01-10T18:01:59.025Z 17 Acked 1016 Read 3 Done Done Sent false
165872024-01-10T18:01:59.025Z 18 Acked 1017 Read 1 Done Done Sent false
165882024-01-10T18:01:59.025Z 19 Acked 1018 Read 3 Done Done Sent false
165892024-01-10T18:01:59.025Z 20 Acked 1019 Write 4 Done Done Sent false
165902024-01-10T18:01:59.025Z 21 Acked 1020 Read 5 Done Done Sent false
165912024-01-10T18:01:59.025Z 22 Acked 1021 Write 7 Done Done Sent false
165922024-01-10T18:01:59.025Z 23 Acked 1022 Write 6 Done Done Sent false
165932024-01-10T18:01:59.026Z 24 Acked 1023 Read 6 Done Done Sent false
165942024-01-10T18:01:59.026Z 25 Acked 1024 Flush 0 Done Done Sent false
165952024-01-10T18:01:59.026Z 26 Acked 1025 Flush 0 Done Done Sent false
165962024-01-10T18:01:59.026Z 27 Acked 1026 Write 9 Done Done Sent false
165972024-01-10T18:01:59.027Z 28 Acked 1027 Write 4 Done Done Sent false
165982024-01-10T18:01:59.027Z 29 Acked 1028 Write 10 Sent Sent Sent false
165992024-01-10T18:01:59.027Z 30 Acked 1029 Write 1 Sent Sent Sent false
166002024-01-10T18:01:59.027Z STATES DS:0 DS:1 DS:2 TOTAL
166012024-01-10T18:01:59.027Z New 0 0 0 0
166022024-01-10T18:01:59.027Z Sent 2 2 26 30
166032024-01-10T18:01:59.028Z Done 28 28 4 60
166042024-01-10T18:01:59.028Z Skipped 0 0 0 0
166052024-01-10T18:01:59.028Z Error 0 0 0 0
166062024-01-10T18:01:59.028ZLast Flush: 1025 1025 0
166072024-01-10T18:01:59.029ZDownstairs last five completed:
166082024-01-10T18:01:59.029ZUpstairs last five completed: 30 29 28 27 26
166092024-01-10T18:01:59.030ZWrote out file "/tmp/test_repair_verify.data"
166102024-01-10T18:01:59.030Z
166112024-01-10T18:01:59.031ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166122024-01-10T18:01:59.032ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166132024-01-10T18:01:59.032Z 0 000-019 151 151 151 270 270 270 F F F
166142024-01-10T18:01:59.032Z 1 020-039 153 153 153 272 272 272 F F F
166152024-01-10T18:01:59.032Z 2 040-059 151 151 151 270 270 270 T T F <---
166162024-01-10T18:01:59.032Z 3 060-079 154 154 153 273 273 272 T T F <---
166172024-01-10T18:01:59.033Z 4 080-099 146 146 146 263 263 263 F F F
166182024-01-10T18:01:59.033Z 5 100-119 149 149 149 267 267 267 F F F
166192024-01-10T18:01:59.033Z 6 120-139 154 154 150 273 273 269 F F F <---
166202024-01-10T18:01:59.034Z 7 140-159 154 154 153 273 273 272 F F T <---
166212024-01-10T18:01:59.034Z 8 160-179 154 154 153 273 273 272 F F F <---
166222024-01-10T18:01:59.035Z 9 180-199 150 150 150 268 268 268 F F F
166232024-01-10T18:01:59.035Z 10 200-219 150 150 150 269 269 269 F F F
166242024-01-10T18:01:59.035Z 11 220-239 149 149 149 267 267 267 F F F
166252024-01-10T18:01:59.035Z 12 240-259 154 154 150 273 273 269 F F T <---
166262024-01-10T18:01:59.035Z 13 260-279 150 150 150 269 269 269 F F F
166272024-01-10T18:01:59.036Z 14 280-299 146 146 146 263 263 263 F F F
166282024-01-10T18:01:59.036Z 15 300-319 154 154 150 273 273 269 F F T <---
166292024-01-10T18:01:59.036Z 16 320-339 154 154 148 273 273 266 T T T <---
166302024-01-10T18:01:59.036Z 17 340-359 153 153 153 272 272 272 F F F
166312024-01-10T18:01:59.037Z 18 360-379 151 151 151 270 270 270 F F F
166322024-01-10T18:01:59.037Z 19 380-399 146 146 146 263 263 263 F F F
166332024-01-10T18:01:59.038Z 20 400-419 153 153 153 272 272 272 F F F
166342024-01-10T18:01:59.038Z 21 420-439 154 154 153 273 273 272 F F T <---
166352024-01-10T18:01:59.038Z 22 440-459 154 154 153 273 273 272 F F T <---
166362024-01-10T18:01:59.038Z 23 460-479 154 154 153 273 273 272 F F F <---
166372024-01-10T18:01:59.038Z 24 480-499 147 147 147 265 265 265 F F F
166382024-01-10T18:01:59.038Z 25 500-519 151 151 151 270 270 270 F F F
166392024-01-10T18:01:59.039Z 26 520-539 154 154 153 273 273 272 F F F <---
166402024-01-10T18:01:59.039Z 27 540-559 154 154 150 273 273 269 T T F <---
166412024-01-10T18:01:59.039Z 28 560-579 154 154 144 273 273 258 F F F <---
166422024-01-10T18:01:59.039Z 29 580-599 154 154 153 273 273 272 F F F <---
166432024-01-10T18:01:59.040ZMax gen: 154, Max flush: 273
166442024-01-10T18:01:59.040ZError: Difference in extent metadata found!
166452024-01-10T18:01:59.040ZA Difference in extent metadata is expected here
166462024-01-10T18:01:59.040ZOn loop 77
166472024-01-10T18:01:59.040Z
166482024-01-10T18:01:59.040ZVerifying data now
166492024-01-10T18:02:05.173ZNo change after verify
166502024-01-10T18:02:05.174ZLoop: 77 Downstairs dump after verify (and repair):
166512024-01-10T18:02:05.191ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166522024-01-10T18:02:05.192Z 0 000-019 151 151 151 270 270 270 F F F
166532024-01-10T18:02:05.192Z 1 020-039 153 153 153 272 272 272 F F F
166542024-01-10T18:02:05.192Z 2 040-059 155 155 155 274 274 274 F F F
166552024-01-10T18:02:05.192Z 3 060-079 155 155 155 274 274 274 F F F
166562024-01-10T18:02:05.192Z 4 080-099 146 146 146 263 263 263 F F F
166572024-01-10T18:02:05.192Z 5 100-119 149 149 149 267 267 267 F F F
166582024-01-10T18:02:05.192Z 6 120-139 154 154 154 273 273 273 F F F
166592024-01-10T18:02:05.192Z 7 140-159 154 154 154 273 273 273 F F F
166602024-01-10T18:02:05.192Z 8 160-179 154 154 154 273 273 273 F F F
166612024-01-10T18:02:05.193Z 9 180-199 150 150 150 268 268 268 F F F
166622024-01-10T18:02:05.193Z 10 200-219 150 150 150 269 269 269 F F F
166632024-01-10T18:02:05.193Z 11 220-239 149 149 149 267 267 267 F F F
166642024-01-10T18:02:05.193Z 12 240-259 154 154 154 273 273 273 F F F
166652024-01-10T18:02:05.193Z 13 260-279 150 150 150 269 269 269 F F F
166662024-01-10T18:02:05.194Z 14 280-299 146 146 146 263 263 263 F F F
166672024-01-10T18:02:05.194Z 15 300-319 154 154 154 273 273 273 F F F
166682024-01-10T18:02:05.194Z 16 320-339 155 155 155 274 274 274 F F F
166692024-01-10T18:02:05.194Z 17 340-359 153 153 153 272 272 272 F F F
166702024-01-10T18:02:05.194Z 18 360-379 151 151 151 270 270 270 F F F
166712024-01-10T18:02:05.194Z 19 380-399 146 146 146 263 263 263 F F F
166722024-01-10T18:02:05.194Z 20 400-419 153 153 153 272 272 272 F F F
166732024-01-10T18:02:05.194Z 21 420-439 154 154 154 273 273 273 F F F
166742024-01-10T18:02:05.195Z 22 440-459 154 154 154 273 273 273 F F F
166752024-01-10T18:02:05.195Z 23 460-479 154 154 154 273 273 273 F F F
166762024-01-10T18:02:05.195Z 24 480-499 147 147 147 265 265 265 F F F
166772024-01-10T18:02:05.195Z 25 500-519 151 151 151 270 270 270 F F F
166782024-01-10T18:02:05.195Z 26 520-539 154 154 154 273 273 273 F F F
166792024-01-10T18:02:05.195Z 27 540-559 155 155 155 274 274 274 F F F
166802024-01-10T18:02:05.195Z 28 560-579 154 154 154 273 273 273 F F F
166812024-01-10T18:02:05.195Z 29 580-599 154 154 154 273 273 273 F F F
166822024-01-10T18:02:05.195ZMax gen: 155, Max flush: 274
166832024-01-10T18:02:05.195Z
166842024-01-10T18:02:05.195ZBegin loop 78
166852024-01-10T18:02:05.196ZDownstairs to restart: 2
166862024-01-10T18:02:05.196Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.181840822Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782}
166872024-01-10T18:02:05.196Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.182394435Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782}
166882024-01-10T18:02:05.197Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.182466537Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782}
166892024-01-10T18:02:05.197Z{"msg":"Crucible b26aef41-b249-45e8-9f5c-583fbff5ffad has session id: cba5bddf-1527-411d-95ea-aa22aecaf60b","v":0,"name":"crucible","level":30Crucible runtime is spawned
166902024-01-10T18:02:05.197Z,"time":"2024-01-10T18:02:05.182503422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
166912024-01-10T18:02:05.197Z{"msg":"Upstairs opts: Upstairs UUID: b26aef41-b249-45e8-9f5c-583fbff5ffad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.182567985Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
166922024-01-10T18:02:05.197Z{"msg":"Crucible stats registered with UUID: b26aef41-b249-45e8-9f5c-583fbff5ffad","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.182634557Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
166932024-01-10T18:02:05.197Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.182697283Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782}
166942024-01-10T18:02:05.197Z{"msg":"b26aef41-b249-45e8-9f5c-583fbff5ffad active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.182975799Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
166952024-01-10T18:02:05.197Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.18303016Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
166962024-01-10T18:02:05.197Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.183057796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
166972024-01-10T18:02:05.197Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.183142251Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
166982024-01-10T18:02:05.197Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.183175301Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
166992024-01-10T18:02:05.198Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.183215471Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167002024-01-10T18:02:05.199Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.183258998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167012024-01-10T18:02:05.199Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.183303421Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167022024-01-10T18:02:05.199Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.183423537Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167032024-01-10T18:02:05.199Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.18350137Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167042024-01-10T18:02:05.199Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.183586104Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167052024-01-10T18:02:05.199Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.183631058Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167062024-01-10T18:02:05.199Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.1836812Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167072024-01-10T18:02:05.209Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.184798497Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167082024-01-10T18:02:05.210Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.184879775Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167092024-01-10T18:02:05.210Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.184926123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167102024-01-10T18:02:05.210Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.18496724Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167112024-01-10T18:02:05.210Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.185841115Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167122024-01-10T18:02:05.210Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.186147878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167132024-01-10T18:02:05.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.18622379Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167142024-01-10T18:02:05.211Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.186269498Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167152024-01-10T18:02:05.211Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.186481368Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167162024-01-10T18:02:05.211Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.186614091Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167172024-01-10T18:02:05.211Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.186641323Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167182024-01-10T18:02:05.211Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.187128929Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167192024-01-10T18:02:05.211Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.187186076Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167202024-01-10T18:02:05.211Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.18721103Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167212024-01-10T18:02:05.212Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.187251287Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167222024-01-10T18:02:05.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.187930399Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167232024-01-10T18:02:05.212Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188009789Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167242024-01-10T18:02:05.212Z{"msg":"[0]R flush_numbers[0..12]: [270, 272, 274, 274, 263, 267, 273, 273, 273, 268, 269, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188069258Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167252024-01-10T18:02:05.212Z{"msg":"[0]R generation[0..12]: [151, 153, 155, 155, 146, 149, 154, 154, 154, 150, 150, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188095996Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167262024-01-10T18:02:05.212Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188138096Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167272024-01-10T18:02:05.212Z{"msg":"[1]R flush_numbers[0..12]: [270, 272, 274, 274, 263, 267, 273, 273, 273, 268, 269, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.18816223Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167282024-01-10T18:02:05.214Z{"msg":"[1]R generation[0..12]: [151, 153, 155, 155, 146, 149, 154, 154, 154, 150, 150, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188203638Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167292024-01-10T18:02:05.214Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188228306Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167302024-01-10T18:02:05.214ZWait for a query_work_queue command to finish before sending IO
167312024-01-10T18:02:05.214Z{"msg":"[2]R flush_numbers[0..12]: [270, 272, 274, 274, 263, 267, 273, 273, 273, 268, 269, 267]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188274196Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167322024-01-10T18:02:05.214Z{"msg":"[2]R generation[0..12]: [151, 153, 155, 155, 146, 149, 154, 154, 154, 150, 150, 149]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188298263Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167332024-01-10T18:02:05.214Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188340663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167342024-01-10T18:02:05.215Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188363292Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167352024-01-10T18:02:05.215Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188403355Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167362024-01-10T18:02:05.215Z{"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188425875Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167372024-01-10T18:02:05.215Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188448641Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167382024-01-10T18:02:05.215Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188487883Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167392024-01-10T18:02:05.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188509108Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167402024-01-10T18:02:05.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.18855447Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167412024-01-10T18:02:05.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188604274Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167422024-01-10T18:02:05.216Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188627356Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167432024-01-10T18:02:05.216Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188665135Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167442024-01-10T18:02:05.216Z{"msg":"b26aef41-b249-45e8-9f5c-583fbff5ffad is now active with session: cba5bddf-1527-411d-95ea-aa22aecaf60b","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188687873Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167452024-01-10T18:02:05.216Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
167462024-01-10T18:02:05.216Z,"time":"2024-01-10T18:02:05.188742311Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782,"session_id":"cba5bddf-1527-411d-95ea-aa22aecaf60b"}
167472024-01-10T18:02:05.216Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:05.188782725Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1782}
167482024-01-10T18:02:05.216ZLoading write count information from file "/tmp/test_repair_verify.data"
167492024-01-10T18:02:05.216ZRun Repair workload
167502024-01-10T18:02:05.217Z01/30 Flush
167512024-01-10T18:02:05.218Z02/30 Write block 91 len 8192 data: 14 14
167522024-01-10T18:02:05.218Z03/30 Read block 394 len 20480
167532024-01-10T18:02:05.218Z04/30 Read block 362 len 8192
167542024-01-10T18:02:05.218Z05/30 Read block 404 len 28672
167552024-01-10T18:02:05.218Z06/30 Read block 95 len 16384
167562024-01-10T18:02:05.218Z07/30 Read block 211 len 32768
167572024-01-10T18:02:05.218Z08/30 Read block 107 len 24576
167582024-01-10T18:02:05.219Z09/30 Write block 553 len 24576 data: 18 18 15 14 12 10
167592024-01-10T18:02:05.639Z10/30 Write block 509 len 32768 data: 14 13 15 16 15 17 16 14
167602024-01-10T18:02:05.639Z11/30 Write block 404 len 24576 data: 15 17 18 18 15 12
167612024-01-10T18:02:05.639Z12/30 Read block 529 len 40960
167622024-01-10T18:02:05.639Z13/30 Read block 200 len 12288
167632024-01-10T18:02:05.639Z14/30 Write block 105 len 20480 data: 6 7 6 7 10
167642024-01-10T18:02:05.639Z15/30 Read block 120 len 8192
167652024-01-10T18:02:05.639Z16/30 Read block 266 len 20480
167662024-01-10T18:02:05.639Z17/30 Write block 244 len 4096 data: 18
167672024-01-10T18:02:05.639Z18/30 Flush
167682024-01-10T18:02:05.639Z19/30 Write block 218 len 36864 data: 11 10 10 12 11 9 7 9 9
167692024-01-10T18:02:05.639Z20/30 Flush
167702024-01-10T18:02:05.644Z21/30 Write block 548 len 40960 data: 17 18 19 18 20 19 19 16 15 13
167712024-01-10T18:02:05.644Z22/30 Write block 75 len 8192 data: 13 13
167722024-01-10T18:02:05.644Z23/30 Read block 144 len 28672
167732024-01-10T18:02:05.645Z24/30 Write block 553 len 8192 data: 20 20
167742024-01-10T18:02:05.645Z25/30 Read block 507 len 20480
167752024-01-10T18:02:05.645Z26/30 Read block 513 len 20480
167762024-01-10T18:02:05.645Z27/30 Write block 315 len 20480 data: 13 12 9 10 10
167772024-01-10T18:02:05.645Z28/30 Write block 137 len 12288 data: 11 12 12
167782024-01-10T18:02:05.645Z29/30 Read block 430 len 16384
167792024-01-10T18:02:05.645Z30/30 Read block 98 len 12288
167802024-01-10T18:02:05.645Z----------------------------------------------------------------
167812024-01-10T18:02:05.645Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:29
167822024-01-10T18:02:05.645ZStates: Active Active Active
167832024-01-10T18:02:05.645ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
167842024-01-10T18:02:05.646Z 2 Acked 1001 Write 2 Done Done Done false
167852024-01-10T18:02:05.646Z 3 Acked 1002 Read 5 Done Done Sent false
167862024-01-10T18:02:05.646Z 4 Acked 1003 Read 2 Done Done Sent false
167872024-01-10T18:02:05.646Z 5 Acked 1004 Read 7 Done Done Sent false
167882024-01-10T18:02:05.646Z 6 Acked 1005 Read 4 Done Done Sent false
167892024-01-10T18:02:05.646Z 7 Acked 1006 Read 8 Done Done Sent false
167902024-01-10T18:02:05.646Z 8 Acked 1007 Read 6 Done Done Sent false
167912024-01-10T18:02:05.646Z 9 Acked 1008 Write 6 Done Done Sent false
167922024-01-10T18:02:05.646Z 10 Acked 1009 Write 8 Done Done Sent false
167932024-01-10T18:02:05.646Z 11 Acked 1010 Write 6 Done Done Sent false
167942024-01-10T18:02:05.646Z 12 Acked 1011 Read 10 Done Done Sent false
167952024-01-10T18:02:05.646Z 13 Acked 1012 Read 3 Done Done Sent false
167962024-01-10T18:02:05.646Z 14 Acked 1013 Write 5 Done Done Sent false
167972024-01-10T18:02:05.646Z 15 Acked 1014 Read 2 Done Done Sent false
167982024-01-10T18:02:05.646Z 16 Acked 1015 Read 5 Done Done Sent false
167992024-01-10T18:02:05.646Z 17 Acked 1016 Write 1 Done Done Sent false
168002024-01-10T18:02:05.646Z 18 Acked 1017 Flush 0 Done Done Sent false
168012024-01-10T18:02:05.646Z 19 Acked 1018 Write 9 Done Done Sent false
168022024-01-10T18:02:05.646Z 20 Acked 1019 Flush 0 Done Done Sent false
168032024-01-10T18:02:05.646Z 21 Acked 1020 Write 10 Done Done Sent false
168042024-01-10T18:02:05.646Z 22 Acked 1021 Write 2 Done Done Sent false
168052024-01-10T18:02:05.646Z 23 Acked 1022 Read 7 Done Done Sent false
168062024-01-10T18:02:05.646Z 24 Acked 1023 Write 2 Done Done Sent false
168072024-01-10T18:02:05.646Z 25 Acked 1024 Read 5 Done Done Sent false
168082024-01-10T18:02:05.647Z 26 Acked 1025 Read 5 Done Done Sent false
168092024-01-10T18:02:05.647Z 27 Acked 1026 Write 5 Done Done Sent false
168102024-01-10T18:02:05.647Z 28 Acked 1027 Write 3 Done Done Sent false
168112024-01-10T18:02:05.647Z 29 Acked 1028 Read 4 Done Done Sent false
168122024-01-10T18:02:05.647Z 30 Acked 1029 Read 3 Done Sent Sent false
168132024-01-10T18:02:05.647Z STATES DS:0 DS:1 DS:2 TOTAL
168142024-01-10T18:02:05.647Z New 0 0 0 0
168152024-01-10T18:02:05.647Z Sent 0 1 28 29
168162024-01-10T18:02:05.647Z Done 29 28 1 58
168172024-01-10T18:02:05.647Z Skipped 0 0 0 0
168182024-01-10T18:02:05.647Z Error 0 0 0 0
168192024-01-10T18:02:05.647ZLast Flush: 1019 1019 1000
168202024-01-10T18:02:05.647ZDownstairs last five completed: 1000
168212024-01-10T18:02:05.647ZUpstairs last five completed: 30 29 28 27 26
168222024-01-10T18:02:05.647ZWrote out file "/tmp/test_repair_verify.data"
168232024-01-10T18:02:05.647Z
168242024-01-10T18:02:05.647ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168252024-01-10T18:02:05.647ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168262024-01-10T18:02:05.647Z 0 000-019 151 151 151 270 270 270 F F F
168272024-01-10T18:02:05.647Z 1 020-039 153 153 153 272 272 272 F F F
168282024-01-10T18:02:05.647Z 2 040-059 155 155 155 274 274 274 F F F
168292024-01-10T18:02:05.647Z 3 060-079 155 155 155 274 274 274 T T F <---
168302024-01-10T18:02:05.647Z 4 080-099 156 156 146 276 276 263 F F T <---
168312024-01-10T18:02:05.647Z 5 100-119 156 156 149 276 276 267 F F F <---
168322024-01-10T18:02:05.647Z 6 120-139 154 154 154 273 273 273 T T F <---
168332024-01-10T18:02:05.648Z 7 140-159 154 154 154 273 273 273 F F F
168342024-01-10T18:02:05.648Z 8 160-179 154 154 154 273 273 273 F F F
168352024-01-10T18:02:05.648Z 9 180-199 150 150 150 268 268 268 F F F
168362024-01-10T18:02:05.648Z 10 200-219 156 156 150 277 277 269 F F F <---
168372024-01-10T18:02:05.648Z 11 220-239 156 156 149 277 277 267 F F F <---
168382024-01-10T18:02:05.648Z 12 240-259 156 156 154 276 276 273 F F F <---
168392024-01-10T18:02:05.648Z 13 260-279 150 150 150 269 269 269 F F F
168402024-01-10T18:02:05.648Z 14 280-299 146 146 146 263 263 263 F F F
168412024-01-10T18:02:05.648Z 15 300-319 154 154 154 273 273 273 T T F <---
168422024-01-10T18:02:05.648Z 16 320-339 155 155 155 274 274 274 F F F
168432024-01-10T18:02:05.648Z 17 340-359 153 153 153 272 272 272 F F F
168442024-01-10T18:02:05.648Z 18 360-379 151 151 151 270 270 270 F F F
168452024-01-10T18:02:05.649Z 19 380-399 146 146 146 263 263 263 F F F
168462024-01-10T18:02:05.649Z 20 400-419 156 156 153 276 276 272 F F F <---
168472024-01-10T18:02:05.649Z 21 420-439 154 154 154 273 273 273 F F F
168482024-01-10T18:02:05.649Z 22 440-459 154 154 154 273 273 273 F F F
168492024-01-10T18:02:05.649Z 23 460-479 154 154 154 273 273 273 F F F
168502024-01-10T18:02:05.649Z 24 480-499 147 147 147 265 265 265 F F F
168512024-01-10T18:02:05.649Z 25 500-519 156 156 151 276 276 270 F F F <---
168522024-01-10T18:02:05.649Z 26 520-539 154 154 154 273 273 273 F F F
168532024-01-10T18:02:05.649Z 27 540-559 156 156 155 276 276 274 T T F <---
168542024-01-10T18:02:05.649Z 28 560-579 154 154 154 273 273 273 F F F
168552024-01-10T18:02:05.649Z 29 580-599 154 154 154 273 273 273 F F F
168562024-01-10T18:02:05.649ZMax gen: 156, Max flush: 277
168572024-01-10T18:02:05.649ZError: Difference in extent metadata found!
168582024-01-10T18:02:05.649ZA Difference in extent metadata is expected here
168592024-01-10T18:02:05.649ZOn loop 78
168602024-01-10T18:02:05.649Z
168612024-01-10T18:02:05.649ZVerifying data now
168622024-01-10T18:02:11.461ZNo change after verify
168632024-01-10T18:02:11.461ZLoop: 78 Downstairs dump after verify (and repair):
168642024-01-10T18:02:11.467ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168652024-01-10T18:02:11.467Z 0 000-019 151 151 151 270 270 270 F F F
168662024-01-10T18:02:11.467Z 1 020-039 153 153 153 272 272 272 F F F
168672024-01-10T18:02:11.467Z 2 040-059 155 155 155 274 274 274 F F F
168682024-01-10T18:02:11.467Z 3 060-079 157 157 157 278 278 278 F F F
168692024-01-10T18:02:11.467Z 4 080-099 156 156 156 276 276 276 F F F
168702024-01-10T18:02:11.467Z 5 100-119 156 156 156 276 276 276 F F F
168712024-01-10T18:02:11.467Z 6 120-139 157 157 157 278 278 278 F F F
168722024-01-10T18:02:11.468Z 7 140-159 154 154 154 273 273 273 F F F
168732024-01-10T18:02:11.469Z 8 160-179 154 154 154 273 273 273 F F F
168742024-01-10T18:02:11.469Z 9 180-199 150 150 150 268 268 268 F F F
168752024-01-10T18:02:11.470Z 10 200-219 156 156 156 277 277 277 F F F
168762024-01-10T18:02:11.470Z 11 220-239 156 156 156 277 277 277 F F F
168772024-01-10T18:02:11.470Z 12 240-259 156 156 156 276 276 276 F F F
168782024-01-10T18:02:11.470Z 13 260-279 150 150 150 269 269 269 F F F
168792024-01-10T18:02:11.470Z 14 280-299 146 146 146 263 263 263 F F F
168802024-01-10T18:02:11.470Z 15 300-319 157 157 157 278 278 278 F F F
168812024-01-10T18:02:11.470Z 16 320-339 155 155 155 274 274 274 F F F
168822024-01-10T18:02:11.470Z 17 340-359 153 153 153 272 272 272 F F F
168832024-01-10T18:02:11.470Z 18 360-379 151 151 151 270 270 270 F F F
168842024-01-10T18:02:11.470Z 19 380-399 146 146 146 263 263 263 F F F
168852024-01-10T18:02:11.470Z 20 400-419 156 156 156 276 276 276 F F F
168862024-01-10T18:02:11.470Z 21 420-439 154 154 154 273 273 273 F F F
168872024-01-10T18:02:11.471Z 22 440-459 154 154 154 273 273 273 F F F
168882024-01-10T18:02:11.471Z 23 460-479 154 154 154 273 273 273 F F F
168892024-01-10T18:02:11.471Z 24 480-499 147 147 147 265 265 265 F F F
168902024-01-10T18:02:11.471Z 25 500-519 156 156 156 276 276 276 F F F
168912024-01-10T18:02:11.471Z 26 520-539 154 154 154 273 273 273 F F F
168922024-01-10T18:02:11.471Z 27 540-559 157 157 157 278 278 278 F F F
168932024-01-10T18:02:11.471Z 28 560-579 154 154 154 273 273 273 F F F
168942024-01-10T18:02:11.471Z 29 580-599 154 154 154 273 273 273 F F F
168952024-01-10T18:02:11.471ZMax gen: 157, Max flush: 278
168962024-01-10T18:02:11.471Z
168972024-01-10T18:02:11.471ZBegin loop 79
168982024-01-10T18:02:11.471ZDownstairs to restart: 0
168992024-01-10T18:02:11.471Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.466915322Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790}
169002024-01-10T18:02:11.471Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.467480862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790}
169012024-01-10T18:02:11.472ZCrucible runtime is spawned
169022024-01-10T18:02:11.472Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.46760813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790}
169032024-01-10T18:02:11.472Z{"msg":"Crucible 13104f1a-931d-4724-9407-f05dc21f9f78 has session id: 327c0bf4-1d88-42a1-85b5-48aeacd0925d","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.467674768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169042024-01-10T18:02:11.472Z{"msg":"Upstairs opts: Upstairs UUID: 13104f1a-931d-4724-9407-f05dc21f9f78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.467770431Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169052024-01-10T18:02:11.472Z{"msg":"Crucible stats registered with UUID: 13104f1a-931d-4724-9407-f05dc21f9f78","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.467814459Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169062024-01-10T18:02:11.472Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.467855142Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790}
169072024-01-10T18:02:11.482Z{"msg":"13104f1a-931d-4724-9407-f05dc21f9f78 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.468081535Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169082024-01-10T18:02:11.484Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.468162628Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169092024-01-10T18:02:11.484Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.468191816Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169102024-01-10T18:02:11.484Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.468269468Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169112024-01-10T18:02:11.484Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.468315155Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169122024-01-10T18:02:11.484Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.468415839Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169132024-01-10T18:02:11.485Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.46868807Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169142024-01-10T18:02:11.485Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.46874098Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169152024-01-10T18:02:11.485Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.468781994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169162024-01-10T18:02:11.485Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.468872143Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169172024-01-10T18:02:11.485Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.468918645Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169182024-01-10T18:02:11.486Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.46899633Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169192024-01-10T18:02:11.486Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.469040577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169202024-01-10T18:02:11.486Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.470299893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169212024-01-10T18:02:11.486Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.470341645Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169222024-01-10T18:02:11.486Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.470366937Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169232024-01-10T18:02:11.486Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.470401747Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169242024-01-10T18:02:11.486Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.471242445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169252024-01-10T18:02:11.486Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.471328287Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169262024-01-10T18:02:11.486Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.471410418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169272024-01-10T18:02:11.486Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.471493553Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169282024-01-10T18:02:11.487Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.472296028Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169292024-01-10T18:02:11.487Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.472356376Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169302024-01-10T18:02:11.487Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.472382248Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169312024-01-10T18:02:11.487Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.472404383Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169322024-01-10T18:02:11.487Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.472445298Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169332024-01-10T18:02:11.488Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.472478371Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169342024-01-10T18:02:11.488Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.472521482Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169352024-01-10T18:02:11.488Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.473212275Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169362024-01-10T18:02:11.488Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.473295209Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169372024-01-10T18:02:11.488Z{"msg":"[0]R flush_numbers[0..12]: [270, 272, 274, 278, 276, 276, 278, 273, 273, 268, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.473370163Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169382024-01-10T18:02:11.488Z{"msg":"[0]R generation[0..12]: [151, 153, 155, 157, 156, 156, 157, 154, 154, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.473441802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169392024-01-10T18:02:11.488Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.473516139Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169402024-01-10T18:02:11.488Z{"msg":"Wait for a query_work_queue command to finish before sending IO
169412024-01-10T18:02:11.488Z[1]R flush_numbers[0..12]: [270, 272, 274, 278, 276, 276, 278, 273, 273, 268, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.47367429Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169422024-01-10T18:02:11.488Z{"msg":"[1]R generation[0..12]: [151, 153, 155, 157, 156, 156, 157, 154, 154, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.473753665Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169432024-01-10T18:02:11.488Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.473830721Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169442024-01-10T18:02:11.488Z{"msg":"[2]R flush_numbers[0..12]: [270, 272, 274, 278, 276, 276, 278, 273, 273, 268, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.473904954Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169452024-01-10T18:02:11.488Z{"msg":"[2]R generation[0..12]: [151, 153, 155, 157, 156, 156, 157, 154, 154, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.47398123Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169462024-01-10T18:02:11.489Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.47406113Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169472024-01-10T18:02:11.489Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474136055Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169482024-01-10T18:02:11.489Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474207525Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169492024-01-10T18:02:11.489ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169502024-01-10T18:02:11.489Z{"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474282046Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169512024-01-10T18:02:11.489Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474347065Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169522024-01-10T18:02:11.489Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474399222Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169532024-01-10T18:02:11.489Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474422803Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169542024-01-10T18:02:11.489Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474464206Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169552024-01-10T18:02:11.489Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474487332Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169562024-01-10T18:02:11.489Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474526546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169572024-01-10T18:02:11.489Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.47454764Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169582024-01-10T18:02:11.490Z{"msg":"13104f1a-931d-4724-9407-f05dc21f9f78 is now active with session: 327c0bf4-1d88-42a1-85b5-48aeacd0925d","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474586623Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169592024-01-10T18:02:11.490Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474607152Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790,"session_id":"327c0bf4-1d88-42a1-85b5-48aeacd0925d"}
169602024-01-10T18:02:11.490Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:11.474645023Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1790}
169612024-01-10T18:02:11.490ZLoading write count information from file "/tmp/test_repair_verify.data"
169622024-01-10T18:02:11.490ZRun Repair workload
169632024-01-10T18:02:11.490Z01/30 Write block 361 len 40960 data: 9 11 12 11 10 9 7 7 7 10
169642024-01-10T18:02:11.490Z02/30 Read block 451 len 32768
169652024-01-10T18:02:11.495Z03/30 Read block 15 len 24576
169662024-01-10T18:02:11.495Z04/30 Flush
169672024-01-10T18:02:11.496Z05/30 Write block 149 len 8192 data: 12 11
169682024-01-10T18:02:11.496Z06/30 Write block 272 len 40960 data: 9 9 14 15 13 12 11 11 12 11
169692024-01-10T18:02:11.496Z07/30 Read block 226 len 24576
169702024-01-10T18:02:11.496Z08/30 Flush
169712024-01-10T18:02:11.496Z09/30 Write block 75 len 24576 data: 14 14 13 9 11 11
169722024-01-10T18:02:11.496Z10/30 Flush
169732024-01-10T18:02:11.503Z11/30 Write block 58 len 4096 data: 15
169742024-01-10T18:02:11.503Z12/30 Write block 133 len 28672 data: 9 9 10 8 12 13 13
169752024-01-10T18:02:11.503Z13/30 Read block 501 len 32768
169762024-01-10T18:02:11.521Z14/30 Write block 509 len 12288 data: 15 14 16
169772024-01-10T18:02:11.521Z15/30 Read block 145 len 40960
169782024-01-10T18:02:11.521Z16/30 Read block 200 len 32768
169792024-01-10T18:02:11.522Z17/30 Flush
169802024-01-10T18:02:11.530Z18/30 Write block 439 len 32768 data: 14 12 14 10 10 9 13 16
169812024-01-10T18:02:11.530Z19/30 Write block 375 len 20480 data: 10 8 7 6 7
169822024-01-10T18:02:11.530Z20/30 Read block 5 len 12288
169832024-01-10T18:02:11.530Z21/30 Write block 532 len 28672 data: 14 15 16 15 16 15 14
169842024-01-10T18:02:11.530Z22/30 Read block 361 len 28672
169852024-01-10T18:02:11.530Z23/30 Read block 194 len 12288
169862024-01-10T18:02:11.530Z24/30 Read block 192 len 20480
169872024-01-10T18:02:11.530Z25/30 Write block 112 len 12288 data: 15 16 13
169882024-01-10T18:02:11.530Z26/30 Write block 359 len 8192 data: 12 9
169892024-01-10T18:02:11.530Z27/30 Read block 395 len 40960
169902024-01-10T18:02:11.538Z28/30 Read block 24 len 20480
169912024-01-10T18:02:11.538Z29/30 Read block 245 len 32768
169922024-01-10T18:02:11.538Z30/30 Read block 428 len 16384
169932024-01-10T18:02:11.538Z----------------------------------------------------------------
169942024-01-10T18:02:11.538Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:26
169952024-01-10T18:02:11.538ZStates: Active Active Active
169962024-01-10T18:02:11.538ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
169972024-01-10T18:02:11.538Z 5 Acked 1004 Write 2 Sent Done Done false
169982024-01-10T18:02:11.538Z 6 Acked 1005 Write 10 Sent Done Done false
169992024-01-10T18:02:11.538Z 7 Acked 1006 Read 6 Sent Done Done false
170002024-01-10T18:02:11.539Z 8 Acked 1007 Flush 0 Sent Done Done false
170012024-01-10T18:02:11.539Z 9 Acked 1008 Write 6 Sent Done Done false
170022024-01-10T18:02:11.539Z 10 Acked 1009 Flush 0 Sent Done Done false
170032024-01-10T18:02:11.539Z 11 Acked 1010 Write 1 Sent Done Done false
170042024-01-10T18:02:11.539Z 12 Acked 1011 Write 7 Sent Done Done false
170052024-01-10T18:02:11.539Z 13 Acked 1012 Read 8 Sent Done Done false
170062024-01-10T18:02:11.539Z 14 Acked 1013 Write 3 Sent Done Done false
170072024-01-10T18:02:11.539Z 15 Acked 1014 Read 10 Sent Done Done false
170082024-01-10T18:02:11.540Z 16 Acked 1015 Read 8 Sent Done Done false
170092024-01-10T18:02:11.540Z 17 Acked 1016 Flush 0 Sent Done Done false
170102024-01-10T18:02:11.540Z 18 Acked 1017 Write 8 Sent Done Done false
170112024-01-10T18:02:11.540Z 19 Acked 1018 Write 5 Sent Done Done false
170122024-01-10T18:02:11.540Z 20 Acked 1019 Read 3 Sent Done Done false
170132024-01-10T18:02:11.540Z 21 Acked 1020 Write 7 Sent Done Done false
170142024-01-10T18:02:11.540Z 22 Acked 1021 Read 7 Sent Done Done false
170152024-01-10T18:02:11.540Z 23 Acked 1022 Read 3 Sent Done Done false
170162024-01-10T18:02:11.540Z 24 Acked 1023 Read 5 Sent Done Done false
170172024-01-10T18:02:11.540Z 25 Acked 1024 Write 3 Sent Done Done false
170182024-01-10T18:02:11.540Z 26 Acked 1025 Write 2 Sent Done Done false
170192024-01-10T18:02:11.540Z 27 Acked 1026 Read 10 Sent Done Done false
170202024-01-10T18:02:11.540Z 28 Acked 1027 Read 5 Sent Done Done false
170212024-01-10T18:02:11.540Z 29 Acked 1028 Read 8 Sent Done Done false
170222024-01-10T18:02:11.540Z 30 Acked 1029 Read 4 Sent Done Done false
170232024-01-10T18:02:11.540Z STATES DS:0 DS:1 DS:2 TOTAL
170242024-01-10T18:02:11.540Z New 0 0 0 0
170252024-01-10T18:02:11.541Z Sent 26 0 0 26
170262024-01-10T18:02:11.541Z Done 0 26 26 52
170272024-01-10T18:02:11.542Z Skipped 0 0 0 0
170282024-01-10T18:02:11.542Z Error 0 0 0 0
170292024-01-10T18:02:11.542ZLast Flush: 1003 1016 1016
170302024-01-10T18:02:11.542ZDownstairs last five completed: 1003 1002 1001 1000
170312024-01-10T18:02:11.542ZUpstairs last five completed: 30 29 28 27 26
170322024-01-10T18:02:11.542ZWrote out file "/tmp/test_repair_verify.data"
170332024-01-10T18:02:11.548Z
170342024-01-10T18:02:11.553ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
170352024-01-10T18:02:11.586ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170362024-01-10T18:02:11.587Z 0 000-019 151 151 151 270 270 270 F F F
170372024-01-10T18:02:11.587Z 1 020-039 153 153 153 272 272 272 F F F
170382024-01-10T18:02:12.010Z 2 040-059 155 158 158 274 282 282 F F F <---
170392024-01-10T18:02:12.010Z 3 060-079 157 158 158 278 281 281 F F F <---
170402024-01-10T18:02:12.010Z 4 080-099 156 158 158 276 281 281 F F F <---
170412024-01-10T18:02:12.010Z 5 100-119 156 156 156 276 276 276 F T T <---
170422024-01-10T18:02:12.010Z 6 120-139 157 158 158 278 282 282 F F F <---
170432024-01-10T18:02:12.010Z 7 140-159 154 158 158 273 280 280 F F F <---
170442024-01-10T18:02:12.010Z 8 160-179 154 154 154 273 273 273 F F F
170452024-01-10T18:02:12.010Z 9 180-199 150 150 150 268 268 268 F F F
170462024-01-10T18:02:12.010Z 10 200-219 156 156 156 277 277 277 F F F
170472024-01-10T18:02:12.011Z 11 220-239 156 156 156 277 277 277 F F F
170482024-01-10T18:02:12.011Z 12 240-259 156 156 156 276 276 276 F F F
170492024-01-10T18:02:12.011Z 13 260-279 150 158 158 269 280 280 F F F <---
170502024-01-10T18:02:12.011Z 14 280-299 146 158 158 263 280 280 F F F <---
170512024-01-10T18:02:12.011Z 15 300-319 157 157 157 278 278 278 F F F
170522024-01-10T18:02:12.012Z 16 320-339 155 155 155 274 274 274 F F F
170532024-01-10T18:02:12.012Z 17 340-359 153 153 153 272 272 272 F T T <---
170542024-01-10T18:02:12.012Z 18 360-379 158 158 158 279 279 279 F T T <---
170552024-01-10T18:02:12.012Z 19 380-399 146 146 146 263 263 263 F F F
170562024-01-10T18:02:12.012Z 20 400-419 156 156 156 276 276 276 F F F
170572024-01-10T18:02:12.012Z 21 420-439 154 154 154 273 273 273 F T T <---
170582024-01-10T18:02:12.012Z 22 440-459 154 154 154 273 273 273 F T T <---
170592024-01-10T18:02:12.012Z 23 460-479 154 154 154 273 273 273 F F F
170602024-01-10T18:02:12.012Z 24 480-499 147 147 147 265 265 265 F F F
170612024-01-10T18:02:12.012Z 25 500-519 156 158 158 276 282 282 F F F <---
170622024-01-10T18:02:12.012Z 26 520-539 154 154 154 273 273 273 F T T <---
170632024-01-10T18:02:12.012Z 27 540-559 157 157 157 278 278 278 F F F
170642024-01-10T18:02:12.012Z 28 560-579 154 154 154 273 273 273 F F F
170652024-01-10T18:02:12.012Z 29 580-599 154 154 154 273 273 273 F F F
170662024-01-10T18:02:12.012ZMax gen: 158, Max flush: 282
170672024-01-10T18:02:12.012ZError: Difference in extent metadata found!
170682024-01-10T18:02:12.012ZA Difference in extent metadata is expected here
170692024-01-10T18:02:12.012ZOn loop 79
170702024-01-10T18:02:12.012Z
170712024-01-10T18:02:12.012ZVerifying data now
170722024-01-10T18:02:16.911ZNo change after verify
170732024-01-10T18:02:16.911ZLoop: 79 Downstairs dump after verify (and repair):
170742024-01-10T18:02:16.930ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170752024-01-10T18:02:16.930Z 0 000-019 151 151 151 270 270 270 F F F
170762024-01-10T18:02:16.931Z 1 020-039 153 153 153 272 272 272 F F F
170772024-01-10T18:02:16.931Z 2 040-059 158 158 158 282 282 282 F F F
170782024-01-10T18:02:16.931Z 3 060-079 158 158 158 281 281 281 F F F
170792024-01-10T18:02:16.931Z 4 080-099 158 158 158 281 281 281 F F F
170802024-01-10T18:02:16.931Z 5 100-119 159 159 159 283 283 283 F F F
170812024-01-10T18:02:16.940Z 6 120-139 158 158 158 282 282 282 F F F
170822024-01-10T18:02:16.940Z 7 140-159 158 158 158 280 280 280 F F F
170832024-01-10T18:02:16.940Z 8 160-179 154 154 154 273 273 273 F F F
170842024-01-10T18:02:16.940Z 9 180-199 150 150 150 268 268 268 F F F
170852024-01-10T18:02:16.940Z 10 200-219 156 156 156 277 277 277 F F F
170862024-01-10T18:02:16.940Z 11 220-239 156 156 156 277 277 277 F F F
170872024-01-10T18:02:16.940Z 12 240-259 156 156 156 276 276 276 F F F
170882024-01-10T18:02:16.941Z 13 260-279 158 158 158 280 280 280 F F F
170892024-01-10T18:02:16.941Z 14 280-299 158 158 158 280 280 280 F F F
170902024-01-10T18:02:16.941Z 15 300-319 157 157 157 278 278 278 F F F
170912024-01-10T18:02:16.941Z 16 320-339 155 155 155 274 274 274 F F F
170922024-01-10T18:02:16.941Z 17 340-359 159 159 159 283 283 283 F F F
170932024-01-10T18:02:16.941Z 18 360-379 159 159 159 283 283 283 F F F
170942024-01-10T18:02:16.941Z 19 380-399 146 146 146 263 263 263 F F F
170952024-01-10T18:02:16.941Z 20 400-419 156 156 156 276 276 276 F F F
170962024-01-10T18:02:16.941Z 21 420-439 159 159 159 283 283 283 F F F
170972024-01-10T18:02:16.941Z 22 440-459 159 159 159 283 283 283 F F F
170982024-01-10T18:02:16.941Z 23 460-479 154 154 154 273 273 273 F F F
170992024-01-10T18:02:16.941Z 24 480-499 147 147 147 265 265 265 F F F
171002024-01-10T18:02:16.941Z 25 500-519 158 158 158 282 282 282 F F F
171012024-01-10T18:02:16.941Z 26 520-539 159 159 159 283 283 283 F F F
171022024-01-10T18:02:16.941Z 27 540-559 157 157 157 278 278 278 F F F
171032024-01-10T18:02:16.941Z 28 560-579 154 154 154 273 273 273 F F F
171042024-01-10T18:02:16.942Z 29 580-599 154 154 154 273 273 273 F F F
171052024-01-10T18:02:16.942ZMax gen: 159, Max flush: 283
171062024-01-10T18:02:16.942Z
171072024-01-10T18:02:16.942ZBegin loop 80
171082024-01-10T18:02:16.942ZDownstairs to restart: 2
171092024-01-10T18:02:16.965Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.966085996Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798}
171102024-01-10T18:02:16.965Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.966653329Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798}
171112024-01-10T18:02:16.969Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
171122024-01-10T18:02:16.970Z,"time":"2024-01-10T18:02:16.966785668Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798}
171132024-01-10T18:02:16.970Z{"msg":"Crucible 9a8577f2-78b7-4ab1-95cf-73fdb1fb0289 has session id: d0533c7c-c088-4d9d-b40c-de180a15550f","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.966920344Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171142024-01-10T18:02:16.970Z{"msg":"Upstairs opts: Upstairs UUID: 9a8577f2-78b7-4ab1-95cf-73fdb1fb0289, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.966997384Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171152024-01-10T18:02:16.970Z{"msg":"Crucible stats registered with UUID: 9a8577f2-78b7-4ab1-95cf-73fdb1fb0289","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.96707816Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171162024-01-10T18:02:16.970Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967104082Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798}
171172024-01-10T18:02:16.970Z{"msg":"9a8577f2-78b7-4ab1-95cf-73fdb1fb0289 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967124706Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171182024-01-10T18:02:16.970Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967182751Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171192024-01-10T18:02:16.972Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967225483Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171202024-01-10T18:02:16.972Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967296939Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171212024-01-10T18:02:16.972Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967341543Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171222024-01-10T18:02:16.972Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967378083Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171232024-01-10T18:02:16.972Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967417884Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171242024-01-10T18:02:16.972Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967491594Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171252024-01-10T18:02:16.972Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967526135Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171262024-01-10T18:02:16.972Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967570178Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171272024-01-10T18:02:16.973Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967698706Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171282024-01-10T18:02:16.973Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967790485Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171292024-01-10T18:02:16.974Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.967866316Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171302024-01-10T18:02:16.974Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.969191705Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171312024-01-10T18:02:16.974Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.969263595Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171322024-01-10T18:02:16.974Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.969290847Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171332024-01-10T18:02:16.974Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.969313984Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171342024-01-10T18:02:16.974Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.969996992Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171352024-01-10T18:02:16.974Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.970143732Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171362024-01-10T18:02:16.974Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.970273611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171372024-01-10T18:02:16.974Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.970364056Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171382024-01-10T18:02:16.979Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.970932207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171392024-01-10T18:02:16.979Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.971260557Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171402024-01-10T18:02:16.979Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.971376737Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171412024-01-10T18:02:16.979Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.971509302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171422024-01-10T18:02:16.979Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.971578748Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171432024-01-10T18:02:16.979Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.971652238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171442024-01-10T18:02:16.979Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.971721474Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171452024-01-10T18:02:16.979Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.971865102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171462024-01-10T18:02:16.979Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.971922389Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171472024-01-10T18:02:16.979Z{"msg":"[0]R flush_numbers[0..12]: [270, 272, 282, 281, 281, 283, 282, 280, 273, 268, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.97194845Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171482024-01-10T18:02:16.979Z{"msg":"[0]R generation[0..12]: [151, 153, 158, 158, 158, 159, 158, 158, 154, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.971972535Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171492024-01-10T18:02:16.979Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.971997091Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171502024-01-10T18:02:16.979Z{"msg":"[1]R flush_numbers[0..12]: [270, 272, 282, 281, 281, 283, 282, 280, 273, 268, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972050756Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171512024-01-10T18:02:16.979Z{"msg":"[1]R generation[0..12]: [151, 153, 158, 158, 158, 159, 158, 158, 154, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972079327Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171522024-01-10T18:02:16.979Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972131958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171532024-01-10T18:02:16.979Z{"msg":"[2]R flush_numbers[0..12]: [270, 272, 282, 281, 281, 283, 282, 280, 273, 268, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972157081Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171542024-01-10T18:02:16.979Z{"msg":"[2]R generation[0..12]: [151, 153, 158, 158, 158, 159, 158, 158, 154, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.97217994Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171552024-01-10T18:02:16.980Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972203669Z","Wait for a query_work_queue command to finish before sending IO
171562024-01-10T18:02:16.980Zhostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171572024-01-10T18:02:16.980Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972286378Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171582024-01-10T18:02:16.980Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972336213Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171592024-01-10T18:02:16.981Z{"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972360059Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171602024-01-10T18:02:16.981Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972381855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171612024-01-10T18:02:16.981Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972404825Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171622024-01-10T18:02:16.981Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972425968Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171632024-01-10T18:02:16.981Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972447937Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171642024-01-10T18:02:16.981Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972469762Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171652024-01-10T18:02:16.981Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972491346Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171662024-01-10T18:02:16.981Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972511484Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171672024-01-10T18:02:16.981Z{"msg":"9a8577f2-78b7-4ab1-95cf-73fdb1fb0289 is now active with session: d0533c7c-c088-4d9d-b40c-de180a15550f","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972556811Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171682024-01-10T18:02:16.981Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972578001Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798,"session_id":"d0533c7c-c088-4d9d-b40c-de180a15550f"}
171692024-01-10T18:02:16.981Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:16.972598218Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1798}
171702024-01-10T18:02:16.982ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
171712024-01-10T18:02:16.983ZLoading write count information from file "/tmp/test_repair_verify.data"
171722024-01-10T18:02:16.983ZRun Repair workload
171732024-01-10T18:02:16.983Z01/30 Write block 367 len 24576 data: 8 8 8 11 10 10
171742024-01-10T18:02:16.983Z02/30 Read block 387 len 16384
171752024-01-10T18:02:16.983Z03/30 Write block 391 len 8192 data: 10 11
171762024-01-10T18:02:16.983Z04/30 Write block 514 len 8192 data: 18 17
171772024-01-10T18:02:16.986Z05/30 Read block 439 len 32768
171782024-01-10T18:02:16.986Z06/30 Read block 312 len 8192
171792024-01-10T18:02:16.986Z07/30 Write block 124 len 36864 data: 9 9 7 7 6 6 8 7 9
171802024-01-10T18:02:16.986Z08/30 Read block 248 len 20480
171812024-01-10T18:02:16.989Z09/30 Flush
171822024-01-10T18:02:16.992Z10/30 Write block 479 len 4096 data: 7
171832024-01-10T18:02:16.992Z11/30 Write block 306 len 36864 data: 9 7 5 5 5 8 5 8 9
171842024-01-10T18:02:16.992Z12/30 Read block 28 len 40960
171852024-01-10T18:02:16.996Z13/30 Write block 137 len 24576 data: 13 14 14 14 15 13
171862024-01-10T18:02:16.999Z14/30 Read block 262 len 16384
171872024-01-10T18:02:17.002Z15/30 Write block 14 len 40960 data: 10 11 11 11 13 12 11 12 10 12
171882024-01-10T18:02:17.002Z16/30 Write block 577 len 24576 data: 13 12 13 16 14 13
171892024-01-10T18:02:17.002Z17/30 Read block 313 len 20480
171902024-01-10T18:02:17.005Z18/30 Read block 10 len 12288
171912024-01-10T18:02:17.008Z19/30 Write block 284 len 28672 data: 12 14 12 12 11 14 15
171922024-01-10T18:02:17.008Z20/30 Flush
171932024-01-10T18:02:17.011Z21/30 Write block 594 len 12288 data: 12 10 8
171942024-01-10T18:02:17.014Z22/30 Write block 263 len 40960 data: 10 11 8 11 13 11 12 13 12 10
171952024-01-10T18:02:17.015Z23/30 Flush
171962024-01-10T18:02:17.017Z24/30 Write block 164 len 24576 data: 16 18 18 18 16 15
171972024-01-10T18:02:17.020Z25/30 Flush
171982024-01-10T18:02:17.023Z26/30 Write block 346 len 28672 data: 14 15 10 10 9 10 12
171992024-01-10T18:02:17.023Z27/30 Read block 174 len 8192
172002024-01-10T18:02:17.023Z28/30 Read block 501 len 20480
172012024-01-10T18:02:17.026Z29/30 Write block 111 len 36864 data: 12 16 17 14 11 13 13 12 15
172022024-01-10T18:02:17.029Z30/30 Read block 414 len 40960
172032024-01-10T18:02:17.032Z----------------------------------------------------------------
172042024-01-10T18:02:17.033Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
172052024-01-10T18:02:17.033ZStates: Active Active Active
172062024-01-10T18:02:17.033ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172072024-01-10T18:02:17.033Z 1 Acked 1000 Write 6 Done Done Done false
172082024-01-10T18:02:17.033Z 2 Acked 1001 Read 4 Done Done Done false
172092024-01-10T18:02:17.033Z 3 Acked 1002 Write 2 Done Done Sent false
172102024-01-10T18:02:17.033Z 4 Acked 1003 Write 2 Done Done Sent false
172112024-01-10T18:02:17.033Z 5 Acked 1004 Read 8 Done Done Sent false
172122024-01-10T18:02:17.033Z 6 Acked 1005 Read 2 Done Done Sent false
172132024-01-10T18:02:17.033Z 7 Acked 1006 Write 9 Done Done Sent false
172142024-01-10T18:02:17.033Z 8 Acked 1007 Read 5 Done Done Sent false
172152024-01-10T18:02:17.033Z 9 Acked 1008 Flush 0 Done Done Sent false
172162024-01-10T18:02:17.033Z 10 Acked 1009 Write 1 Done Done Sent false
172172024-01-10T18:02:17.033Z 11 Acked 1010 Write 9 Done Done Sent false
172182024-01-10T18:02:17.033Z 12 Acked 1011 Read 10 Done Done Sent false
172192024-01-10T18:02:17.033Z 13 Acked 1012 Write 6 Done Done Sent false
172202024-01-10T18:02:17.033Z 14 Acked 1013 Read 4 Done Done Sent false
172212024-01-10T18:02:17.033Z 15 Acked 1014 Write 10 Done Done Sent false
172222024-01-10T18:02:17.033Z 16 Acked 1015 Write 6 Done Done Sent false
172232024-01-10T18:02:17.033Z 17 Acked 1016 Read 5 Done Done Sent false
172242024-01-10T18:02:17.033Z 18 Acked 1017 Read 3 Done Done Sent false
172252024-01-10T18:02:17.033Z 19 Acked 1018 Write 7 Done Done Sent false
172262024-01-10T18:02:17.033Z 20 Acked 1019 Flush 0 Done Done Sent false
172272024-01-10T18:02:17.034Z 21 Acked 1020 Write 3 Done Done Sent false
172282024-01-10T18:02:17.034Z 22 Acked 1021 Write 10 Done Done Sent false
172292024-01-10T18:02:17.034Z 23 Acked 1022 Flush 0 Done Done Sent false
172302024-01-10T18:02:17.034Z 24 Acked 1023 Write 6 Done Done Sent false
172312024-01-10T18:02:17.034Z 25 Acked 1024 Flush 0 Done Done Sent false
172322024-01-10T18:02:17.034Z 26 Acked 1025 Write 7 Done Done Sent false
172332024-01-10T18:02:17.034Z 27 Acked 1026 Read 2 Done Done Sent false
172342024-01-10T18:02:17.034Z 28 Acked 1027 Read 5 Done Done Sent false
172352024-01-10T18:02:17.034Z 29 Acked 1028 Write 9 Done Done Sent false
172362024-01-10T18:02:17.034Z 30 Acked 1029 Read 10 Done Sent Sent false
172372024-01-10T18:02:17.034Z STATES DS:0 DS:1 DS:2 TOTAL
172382024-01-10T18:02:17.034Z New 0 0 0 0
172392024-01-10T18:02:17.034Z Sent 0 1 28 29
172402024-01-10T18:02:17.034Z Done 30 29 2 61
172412024-01-10T18:02:17.035Z Skipped 0 0 0 0
172422024-01-10T18:02:17.035Z Error 0 0 0 0
172432024-01-10T18:02:17.035ZLast Flush: 1024 1024 0
172442024-01-10T18:02:17.035ZDownstairs last five completed:
172452024-01-10T18:02:17.035ZUpstairs last five completed: 30 29 28 27 26
172462024-01-10T18:02:17.035ZWrote out file "/tmp/test_repair_verify.data"
172472024-01-10T18:02:17.039Z
172482024-01-10T18:02:17.045ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
172492024-01-10T18:02:17.068ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172502024-01-10T18:02:17.068Z 0 000-019 160 160 151 285 285 270 F F F <---
172512024-01-10T18:02:17.072Z 1 020-039 160 160 153 285 285 272 F F F <---
172522024-01-10T18:02:17.072Z 2 040-059 158 158 158 282 282 282 F F F
172532024-01-10T18:02:17.072Z 3 060-079 158 158 158 281 281 281 F F F
172542024-01-10T18:02:17.072Z 4 080-099 158 158 158 281 281 281 F F F
172552024-01-10T18:02:17.072Z 5 100-119 159 159 159 283 283 283 T T F <---
172562024-01-10T18:02:17.072Z 6 120-139 160 160 158 285 285 282 F F F <---
172572024-01-10T18:02:17.072Z 7 140-159 160 160 158 285 285 280 F F F <---
172582024-01-10T18:02:17.073Z 8 160-179 160 160 154 287 287 273 F F F <---
172592024-01-10T18:02:17.073Z 9 180-199 150 150 150 268 268 268 F F F
172602024-01-10T18:02:17.073Z 10 200-219 156 156 156 277 277 277 F F F
172612024-01-10T18:02:17.073Z 11 220-239 156 156 156 277 277 277 F F F
172622024-01-10T18:02:17.073Z 12 240-259 156 156 156 276 276 276 F F F
172632024-01-10T18:02:17.073Z 13 260-279 160 160 158 286 286 280 F F F <---
172642024-01-10T18:02:17.073Z 14 280-299 160 160 158 285 285 280 F F F <---
172652024-01-10T18:02:17.073Z 15 300-319 160 160 157 285 285 278 F F F <---
172662024-01-10T18:02:17.073Z 16 320-339 155 155 155 274 274 274 F F F
172672024-01-10T18:02:17.073Z 17 340-359 159 159 159 283 283 283 T T F <---
172682024-01-10T18:02:17.073Z 18 360-379 160 160 159 284 284 283 F F T <---
172692024-01-10T18:02:17.073Z 19 380-399 160 160 146 284 284 263 F F F <---
172702024-01-10T18:02:17.073Z 20 400-419 156 156 156 276 276 276 F F F
172712024-01-10T18:02:17.073Z 21 420-439 159 159 159 283 283 283 F F F
172722024-01-10T18:02:17.073Z 22 440-459 159 159 159 283 283 283 F F F
172732024-01-10T18:02:17.073Z 23 460-479 160 160 154 285 285 273 F F F <---
172742024-01-10T18:02:17.073Z 24 480-499 147 147 147 265 265 265 F F F
172752024-01-10T18:02:17.073Z 25 500-519 160 160 158 284 284 282 F F F <---
172762024-01-10T18:02:17.073Z 26 520-539 159 159 159 283 283 283 F F F
172772024-01-10T18:02:17.073Z 27 540-559 157 157 157 278 278 278 F F F
172782024-01-10T18:02:17.073Z 28 560-579 160 160 154 285 285 273 F F F <---
172792024-01-10T18:02:17.073Z 29 580-599 160 160 154 286 286 273 F F F <---
172802024-01-10T18:02:17.073ZMax gen: 160, Max flush: 287
172812024-01-10T18:02:17.073ZError: Difference in extent metadata found!
172822024-01-10T18:02:17.081ZA Difference in extent metadata is expected here
172832024-01-10T18:02:17.081ZOn loop 80
172842024-01-10T18:02:17.081Z
172852024-01-10T18:02:17.081ZVerifying data now
172862024-01-10T18:02:22.623ZNo change after verify
172872024-01-10T18:02:22.623ZLoop: 80 Downstairs dump after verify (and repair):
172882024-01-10T18:02:22.628ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172892024-01-10T18:02:22.628Z 0 000-019 160 160 160 285 285 285 F F F
172902024-01-10T18:02:22.628Z 1 020-039 160 160 160 285 285 285 F F F
172912024-01-10T18:02:22.628Z 2 040-059 158 158 158 282 282 282 F F F
172922024-01-10T18:02:22.628Z 3 060-079 158 158 158 281 281 281 F F F
172932024-01-10T18:02:22.628Z 4 080-099 158 158 158 281 281 281 F F F
172942024-01-10T18:02:22.628Z 5 100-119 161 161 161 288 288 288 F F F
172952024-01-10T18:02:22.628Z 6 120-139 160 160 160 285 285 285 F F F
172962024-01-10T18:02:22.628Z 7 140-159 160 160 160 285 285 285 F F F
172972024-01-10T18:02:22.628Z 8 160-179 160 160 160 287 287 287 F F F
172982024-01-10T18:02:22.628Z 9 180-199 150 150 150 268 268 268 F F F
172992024-01-10T18:02:22.628Z 10 200-219 156 156 156 277 277 277 F F F
173002024-01-10T18:02:22.628Z 11 220-239 156 156 156 277 277 277 F F F
173012024-01-10T18:02:22.628Z 12 240-259 156 156 156 276 276 276 F F F
173022024-01-10T18:02:22.628Z 13 260-279 160 160 160 286 286 286 F F F
173032024-01-10T18:02:22.628Z 14 280-299 160 160 160 285 285 285 F F F
173042024-01-10T18:02:22.629Z 15 300-319 160 160 160 285 285 285 F F F
173052024-01-10T18:02:22.629Z 16 320-339 155 155 155 274 274 274 F F F
173062024-01-10T18:02:22.629Z 17 340-359 161 161 161 288 288 288 F F F
173072024-01-10T18:02:22.629Z 18 360-379 160 160 160 284 284 284 F F F
173082024-01-10T18:02:22.629Z 19 380-399 160 160 160 284 284 284 F F F
173092024-01-10T18:02:22.629Z 20 400-419 156 156 156 276 276 276 F F F
173102024-01-10T18:02:22.629Z 21 420-439 159 159 159 283 283 283 F F F
173112024-01-10T18:02:22.629Z 22 440-459 159 159 159 283 283 283 F F F
173122024-01-10T18:02:22.629Z 23 460-479 160 160 160 285 285 285 F F F
173132024-01-10T18:02:22.629Z 24 480-499 147 147 147 265 265 265 F F F
173142024-01-10T18:02:22.629Z 25 500-519 160 160 160 284 284 284 F F F
173152024-01-10T18:02:22.629Z 26 520-539 159 159 159 283 283 283 F F F
173162024-01-10T18:02:22.629Z 27 540-559 157 157 157 278 278 278 F F F
173172024-01-10T18:02:22.629Z 28 560-579 160 160 160 285 285 285 F F F
173182024-01-10T18:02:22.629Z 29 580-599 160 160 160 286 286 286 F F F
173192024-01-10T18:02:22.629ZMax gen: 161, Max flush: 288
173202024-01-10T18:02:22.629Z
173212024-01-10T18:02:22.629ZBegin loop 81
173222024-01-10T18:02:22.629ZDownstairs to restart: 1
173232024-01-10T18:02:22.629Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.62665074Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806}
173242024-01-10T18:02:22.629Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.627209957Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806}
173252024-01-10T18:02:22.629Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.62728354Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806}
173262024-01-10T18:02:22.629Z{"msg":"Crucible 5bbd2461-81a1-4eef-b893-ef833d6f4165 has session id: 68aaeb52-38d9-4eb8-8066-355eeb96fea7","v":0,"name":"crucible","level":30Crucible runtime is spawned
173272024-01-10T18:02:22.629Z,"time":"2024-01-10T18:02:22.627321352Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173282024-01-10T18:02:22.630Z{"msg":"Upstairs opts: Upstairs UUID: 5bbd2461-81a1-4eef-b893-ef833d6f4165, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.627478845Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173292024-01-10T18:02:22.630Z{"msg":"Crucible stats registered with UUID: 5bbd2461-81a1-4eef-b893-ef833d6f4165","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.627528197Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173302024-01-10T18:02:22.630Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.62758029Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806}
173312024-01-10T18:02:22.635Z{"msg":"5bbd2461-81a1-4eef-b893-ef833d6f4165 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.627988085Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173322024-01-10T18:02:22.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.62805028Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173332024-01-10T18:02:22.635Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628083923Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173342024-01-10T18:02:22.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628161126Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173352024-01-10T18:02:22.635Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628207232Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173362024-01-10T18:02:22.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628280618Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173372024-01-10T18:02:22.635Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628328344Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173382024-01-10T18:02:22.635Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628398936Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173392024-01-10T18:02:22.635Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628472647Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173402024-01-10T18:02:22.635Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628515266Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173412024-01-10T18:02:22.635Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628602949Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173422024-01-10T18:02:22.636Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628648584Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173432024-01-10T18:02:22.636Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.628692858Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173442024-01-10T18:02:22.636Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.629782096Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173452024-01-10T18:02:22.636Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.629846976Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173462024-01-10T18:02:22.636Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.630079293Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173472024-01-10T18:02:22.636Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.630163147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173482024-01-10T18:02:22.636Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.630191959Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173492024-01-10T18:02:22.636Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.630214219Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173502024-01-10T18:02:22.636Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.630661174Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173512024-01-10T18:02:22.640Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.631257274Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173522024-01-10T18:02:22.640Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.631478669Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173532024-01-10T18:02:22.640Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.631541886Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173542024-01-10T18:02:22.640Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.631568301Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173552024-01-10T18:02:22.640Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.631616678Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173562024-01-10T18:02:22.640Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632229193Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173572024-01-10T18:02:22.640Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632323353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173582024-01-10T18:02:22.640Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632395314Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173592024-01-10T18:02:22.640Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632437183Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173602024-01-10T18:02:22.640Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632490698Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173612024-01-10T18:02:22.640Z{"msg":"[0]R flush_numbers[0..12]: [285, 285, 282, 281, 281, 288, 285, 285, 287, 268, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632535416Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173622024-01-10T18:02:22.641Z{"msg":"[0]R generation[0..12]: [160, 160, 158, 158, 158, 161, 160, 160, 160, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632594973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
173632024-01-10T18:02:22.641Z68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173642024-01-10T18:02:22.641Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632644898Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173652024-01-10T18:02:22.641Z{"msg":"[1]R flush_numbers[0..12]: [285, 285, 282, 281, 281, 288, 285, 285, 287, 268, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632690309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173662024-01-10T18:02:22.641Z{"msg":"[1]R generation[0..12]: [160, 160, 158, 158, 158, 161, 160, 160, 160, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632746275Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173672024-01-10T18:02:22.642Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632791316Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173682024-01-10T18:02:22.642Z{"msg":"[2]R flush_numbers[0..12]: [285, 285, 282, 281, 281, 288, 285, 285, 287, 268, 277, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632844569Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173692024-01-10T18:02:22.642Z{"msg":"[2]R generation[0..12]: [160, 160, 158, 158, 158, 161, 160, 160, 160, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632888061Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173702024-01-10T18:02:22.642Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632948741Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173712024-01-10T18:02:22.642Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.632990839Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173722024-01-10T18:02:22.643Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.633021309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173732024-01-10T18:02:22.643Z{"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.633043213Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173742024-01-10T18:02:22.643Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.633064514Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173752024-01-10T18:02:22.643Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.63310583Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173762024-01-10T18:02:22.643Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.633127587Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173772024-01-10T18:02:22.643Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.633167376Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173782024-01-10T18:02:22.643Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.633190415Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173792024-01-10T18:02:22.643Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.633232984Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173802024-01-10T18:02:22.643Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.633254323Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173812024-01-10T18:02:22.644Z{"msg":"5bbd2461-81a1-4eef-b893-ef833d6f4165 is now active with session: 68aaeb52-38d9-4eb8-8066-355eeb96fea7","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.633295948Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173822024-01-10T18:02:22.644Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.633316772Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1806,"session_id":"68aaeb52-38d9-4eb8-8066-355eeb96fea7"}
173832024-01-10T18:02:22.644Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:22.63335877Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
173842024-01-10T18:02:22.644Zpid":1806}
173852024-01-10T18:02:22.644ZLoading write count information from file "/tmp/test_repair_verify.data"
173862024-01-10T18:02:22.644ZRun Repair workload
173872024-01-10T18:02:22.644Z01/30 Flush
173882024-01-10T18:02:22.650Z02/30 Write block 571 len 20480 data: 10 9 10 14 14
173892024-01-10T18:02:22.650Z03/30 Read block 484 len 20480
173902024-01-10T18:02:22.650Z04/30 Write block 287 len 8192 data: 13 12
173912024-01-10T18:02:22.650Z05/30 Write block 198 len 24576 data: 17 17 16 15 14 11
173922024-01-10T18:02:22.650Z06/30 Read block 471 len 12288
173932024-01-10T18:02:22.650Z07/30 Write block 293 len 28672 data: 15 13 14 13 17 20 16
173942024-01-10T18:02:22.650Z08/30 Flush
173952024-01-10T18:02:22.650Z09/30 Write block 264 len 36864 data: 12 9 12 14 12 13 14 13 11
173962024-01-10T18:02:22.651Z10/30 Write block 117 len 4096 data: 14
173972024-01-10T18:02:23.068Z11/30 Write block 469 len 16384 data: 10 10 7 6
173982024-01-10T18:02:23.069Z12/30 Flush
173992024-01-10T18:02:23.069Z13/30 Write block 87 len 8192 data: 9 7
174002024-01-10T18:02:23.070Z14/30 Write block 403 len 20480 data: 14 16 18 19 19
174012024-01-10T18:02:23.070Z15/30 Read block 228 len 8192
174022024-01-10T18:02:23.091Z16/30 Write block 157 len 28672 data: 13 11 10 12 15 16 14
174032024-01-10T18:02:23.091Z17/30 Write block 196 len 16384 data: 17 17 18 18
174042024-01-10T18:02:23.092Z18/30 Write block 366 len 8192 data: 10 9
174052024-01-10T18:02:23.092Z19/30 Write block 486 len 16384 data: 11 10 11 12
174062024-01-10T18:02:23.092Z20/30 Write block 497 len 8192 data: 16 16
174072024-01-10T18:02:23.092Z21/30 Write block 171 len 16384 data: 10 9 9 10
174082024-01-10T18:02:23.092Z22/30 Flush
174092024-01-10T18:02:23.092Z23/30 Flush
174102024-01-10T18:02:23.092Z24/30 Flush
174112024-01-10T18:02:23.092Z25/30 Write block 54 len 8192 data: 13 12
174122024-01-10T18:02:23.093Z26/30 Write block 143 len 16384 data: 11 11 12 12
174132024-01-10T18:02:23.093Z27/30 Read block 274 len 4096
174142024-01-10T18:02:23.093Z28/30 Read block 571 len 8192
174152024-01-10T18:02:23.093Z29/30 Write block 22 len 32768 data: 11 13 10 11 11 10 11 12
174162024-01-10T18:02:23.093Z30/30 Write block 256 len 40960 data: 9 10 14 14 13 12 10 11 13 10
174172024-01-10T18:02:23.093Z----------------------------------------------------------------
174182024-01-10T18:02:23.093Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:29
174192024-01-10T18:02:23.093ZStates: Active Active Active
174202024-01-10T18:02:23.094ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174212024-01-10T18:02:23.094Z 2 Acked 1001 Write 5 Done Done Done false
174222024-01-10T18:02:23.094Z 3 Acked 1002 Read 5 Done Done Done false
174232024-01-10T18:02:23.094Z 4 Acked 1003 Write 2 Done Done Done false
174242024-01-10T18:02:23.094Z 5 Acked 1004 Write 6 Done Sent Done false
174252024-01-10T18:02:23.094Z 6 Acked 1005 Read 3 Done Sent Done false
174262024-01-10T18:02:23.094Z 7 Acked 1006 Write 7 Done Sent Done false
174272024-01-10T18:02:23.094Z 8 Acked 1007 Flush 0 Done Sent Done false
174282024-01-10T18:02:23.094Z 9 Acked 1008 Write 9 Done Sent Done false
174292024-01-10T18:02:23.094Z 10 Acked 1009 Write 1 Done Sent Done false
174302024-01-10T18:02:23.094Z 11 Acked 1010 Write 4 Done Sent Done false
174312024-01-10T18:02:23.094Z 12 Acked 1011 Flush 0 Done Sent Done false
174322024-01-10T18:02:23.095Z 13 Acked 1012 Write 2 Done Sent Done false
174332024-01-10T18:02:23.095Z 14 Acked 1013 Write 5 Done Sent Done false
174342024-01-10T18:02:23.095Z 15 Acked 1014 Read 2 Done Sent Done false
174352024-01-10T18:02:23.095Z 16 Acked 1015 Write 7 Done Sent Done false
174362024-01-10T18:02:23.095Z 17 Acked 1016 Write 4 Done Sent Done false
174372024-01-10T18:02:23.095Z 18 Acked 1017 Write 2 Done Sent Done false
174382024-01-10T18:02:23.095Z 19 Acked 1018 Write 4 Done Sent Done false
174392024-01-10T18:02:23.095Z 20 Acked 1019 Write 2 Done Sent Done false
174402024-01-10T18:02:23.095Z 21 Acked 1020 Write 4 Done Sent Done false
174412024-01-10T18:02:23.095Z 22 Acked 1021 Flush 0 Done Sent Done false
174422024-01-10T18:02:23.096Z 23 Acked 1022 Flush 0 Done Sent Done false
174432024-01-10T18:02:23.096Z 24 Acked 1023 Flush 0 Done Sent Done false
174442024-01-10T18:02:23.096Z 25 Acked 1024 Write 2 Done Sent Done false
174452024-01-10T18:02:23.096Z 26 Acked 1025 Write 4 Done Sent Done false
174462024-01-10T18:02:23.096Z 27 Acked 1026 Read 1 Done Sent Done false
174472024-01-10T18:02:23.096Z 28 Acked 1027 Read 2 Done Sent Done false
174482024-01-10T18:02:23.096Z 29 Acked 1028 Write 8 Done Sent Done false
174492024-01-10T18:02:23.096Z 30 Acked 1029 Write 10 Sent Sent Sent false
174502024-01-10T18:02:23.096Z STATES DS:0 DS:1 DS:2 TOTAL
174512024-01-10T18:02:23.096Z New 0 0 0 0
174522024-01-10T18:02:23.096Z Sent 1 26 1 28
174532024-01-10T18:02:23.097Z Done 28 3 28 59
174542024-01-10T18:02:23.097Z Skipped 0 0 0 0
174552024-01-10T18:02:23.097Z Error 0 0 0 0
174562024-01-10T18:02:23.097ZLast Flush: 1023 1000 1023
174572024-01-10T18:02:23.097ZDownstairs last five completed: 1000
174582024-01-10T18:02:23.097ZUpstairs last five completed: 30 29 28 27 26
174592024-01-10T18:02:23.097ZWrote out file "/tmp/test_repair_verify.data"
174602024-01-10T18:02:23.097Z
174612024-01-10T18:02:23.097ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
174622024-01-10T18:02:23.097ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174632024-01-10T18:02:23.097Z 0 000-019 160 160 160 285 285 285 F F F
174642024-01-10T18:02:23.097Z 1 020-039 160 160 160 285 285 285 T F T <---
174652024-01-10T18:02:23.097Z 2 040-059 158 158 158 282 282 282 T F T <---
174662024-01-10T18:02:23.098Z 3 060-079 158 158 158 281 281 281 F F F
174672024-01-10T18:02:23.098Z 4 080-099 162 158 162 292 281 292 F F F <---
174682024-01-10T18:02:23.098Z 5 100-119 162 161 162 291 288 291 F F F <---
174692024-01-10T18:02:23.098Z 6 120-139 160 160 160 285 285 285 F F F
174702024-01-10T18:02:23.098Z 7 140-159 162 160 162 292 285 292 T F T <---
174712024-01-10T18:02:23.098Z 8 160-179 162 160 162 292 287 292 F F F <---
174722024-01-10T18:02:23.098Z 9 180-199 162 150 162 292 268 292 F F F <---
174732024-01-10T18:02:23.098Z 10 200-219 162 156 162 290 277 290 F F F <---
174742024-01-10T18:02:23.098Z 11 220-239 156 156 156 277 277 277 F F F
174752024-01-10T18:02:23.098Z 12 240-259 156 156 156 276 276 276 T F T <---
174762024-01-10T18:02:23.098Z 13 260-279 162 160 162 291 286 291 T F T <---
174772024-01-10T18:02:23.098Z 14 280-299 162 160 162 290 285 290 F T F <---
174782024-01-10T18:02:23.098Z 15 300-319 160 160 160 285 285 285 F F F
174792024-01-10T18:02:23.099Z 16 320-339 155 155 155 274 274 274 F F F
174802024-01-10T18:02:23.099Z 17 340-359 161 161 161 288 288 288 F F F
174812024-01-10T18:02:23.099Z 18 360-379 162 160 162 292 284 292 F F F <---
174822024-01-10T18:02:23.099Z 19 380-399 160 160 160 284 284 284 F F F
174832024-01-10T18:02:23.099Z 20 400-419 162 156 162 292 276 292 F F F <---
174842024-01-10T18:02:23.099Z 21 420-439 159 159 159 283 283 283 F F F
174852024-01-10T18:02:23.099Z 22 440-459 159 159 159 283 283 283 F F F
174862024-01-10T18:02:23.099Z 23 460-479 162 160 162 291 285 291 F F F <---
174872024-01-10T18:02:23.099Z 24 480-499 162 147 162 292 265 292 F F F <---
174882024-01-10T18:02:23.099Z 25 500-519 160 160 160 284 284 284 F F F
174892024-01-10T18:02:23.099Z 26 520-539 159 159 159 283 283 283 F F F
174902024-01-10T18:02:23.099Z 27 540-559 157 157 157 278 278 278 F F F
174912024-01-10T18:02:23.100Z 28 560-579 162 160 162 290 285 290 F T F <---
174922024-01-10T18:02:23.100Z 29 580-599 160 160 160 286 286 286 F F F
174932024-01-10T18:02:23.100ZMax gen: 162, Max flush: 292
174942024-01-10T18:02:23.100ZError: Difference in extent metadata found!
174952024-01-10T18:02:23.100ZA Difference in extent metadata is expected here
174962024-01-10T18:02:23.101ZOn loop 81
174972024-01-10T18:02:23.101Z
174982024-01-10T18:02:23.101ZVerifying data now
174992024-01-10T18:02:28.230ZNo change after verify
175002024-01-10T18:02:28.230ZLoop: 81 Downstairs dump after verify (and repair):
175012024-01-10T18:02:28.252ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175022024-01-10T18:02:28.253Z 0 000-019 160 160 160 285 285 285 F F F
175032024-01-10T18:02:28.253Z 1 020-039 163 163 163 293 293 293 F F F
175042024-01-10T18:02:28.253Z 2 040-059 163 163 163 293 293 293 F F F
175052024-01-10T18:02:28.253Z 3 060-079 158 158 158 281 281 281 F F F
175062024-01-10T18:02:28.253Z 4 080-099 162 162 162 292 292 292 F F F
175072024-01-10T18:02:28.253Z 5 100-119 162 162 162 291 291 291 F F F
175082024-01-10T18:02:28.257Z 6 120-139 160 160 160 285 285 285 F F F
175092024-01-10T18:02:28.257Z 7 140-159 163 163 163 293 293 293 F F F
175102024-01-10T18:02:28.257Z 8 160-179 162 162 162 292 292 292 F F F
175112024-01-10T18:02:28.257Z 9 180-199 162 162 162 292 292 292 F F F
175122024-01-10T18:02:28.257Z 10 200-219 162 162 162 290 290 290 F F F
175132024-01-10T18:02:28.257Z 11 220-239 156 156 156 277 277 277 F F F
175142024-01-10T18:02:28.258Z 12 240-259 163 163 163 293 293 293 F F F
175152024-01-10T18:02:28.258Z 13 260-279 163 163 163 293 293 293 F F F
175162024-01-10T18:02:28.259Z 14 280-299 162 162 162 290 290 290 F F F
175172024-01-10T18:02:28.259Z 15 300-319 160 160 160 285 285 285 F F F
175182024-01-10T18:02:28.259Z 16 320-339 155 155 155 274 274 274 F F F
175192024-01-10T18:02:28.259Z 17 340-359 161 161 161 288 288 288 F F F
175202024-01-10T18:02:28.259Z 18 360-379 162 162 162 292 292 292 F F F
175212024-01-10T18:02:28.259Z 19 380-399 160 160 160 284 284 284 F F F
175222024-01-10T18:02:28.259Z 20 400-419 162 162 162 292 292 292 F F F
175232024-01-10T18:02:28.260Z 21 420-439 159 159 159 283 283 283 F F F
175242024-01-10T18:02:28.260Z 22 440-459 159 159 159 283 283 283 F F F
175252024-01-10T18:02:28.260Z 23 460-479 162 162 162 291 291 291 F F F
175262024-01-10T18:02:28.260Z 24 480-499 162 162 162 292 292 292 F F F
175272024-01-10T18:02:28.260Z 25 500-519 160 160 160 284 284 284 F F F
175282024-01-10T18:02:28.260Z 26 520-539 159 159 159 283 283 283 F F F
175292024-01-10T18:02:28.260Z 27 540-559 157 157 157 278 278 278 F F F
175302024-01-10T18:02:28.260Z 28 560-579 162 162 162 290 290 290 F F F
175312024-01-10T18:02:28.260Z 29 580-599 160 160 160 286 286 286 F F F
175322024-01-10T18:02:28.260ZMax gen: 163, Max flush: 293
175332024-01-10T18:02:28.261Z
175342024-01-10T18:02:28.261ZBegin loop 82
175352024-01-10T18:02:28.261ZDownstairs to restart: 1
175362024-01-10T18:02:28.289Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.28550654Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814}
175372024-01-10T18:02:28.292Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.285998817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814}
175382024-01-10T18:02:28.292Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286028101Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814}
175392024-01-10T18:02:28.298Z{"msg":"Crucible ebeb499c-2929-40c9-bf6f-4d883cfa0121 has session id: 00216365-f3ad-41b5-b796-c9670b6b1f94","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286155677Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175402024-01-10T18:02:28.298Z{"msg":"Upstairs opts: Upstairs UUID: ebeb499c-2929-40c9-bf6f-4d883cfa0121, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286243317Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175412024-01-10T18:02:28.298Z{"msg":"Crucible runtime is spawned
175422024-01-10T18:02:28.298ZCrucible stats registered with UUID: ebeb499c-2929-40c9-bf6f-4d883cfa0121","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286297364Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175432024-01-10T18:02:28.298Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286513053Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814}
175442024-01-10T18:02:28.298Z{"msg":"ebeb499c-2929-40c9-bf6f-4d883cfa0121 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.28657211Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175452024-01-10T18:02:28.298Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.28659635Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175462024-01-10T18:02:28.298Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286620494Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175472024-01-10T18:02:28.298Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286643084Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175482024-01-10T18:02:28.298Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286665897Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175492024-01-10T18:02:28.299Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286828225Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175502024-01-10T18:02:28.299Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286874657Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175512024-01-10T18:02:28.299Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.286954865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175522024-01-10T18:02:28.299Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.287097299Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175532024-01-10T18:02:28.299Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.287168001Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175542024-01-10T18:02:28.299Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.287328241Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175552024-01-10T18:02:28.299Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.287440748Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175562024-01-10T18:02:28.299Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.287469813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175572024-01-10T18:02:28.299Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.288899623Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175582024-01-10T18:02:28.299Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.288977598Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175592024-01-10T18:02:28.299Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.289003549Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175602024-01-10T18:02:28.299Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.289108703Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175612024-01-10T18:02:28.299Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.289202335Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175622024-01-10T18:02:28.300Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.289276154Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175632024-01-10T18:02:28.300Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.289890881Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175642024-01-10T18:02:28.300Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.290263972Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175652024-01-10T18:02:28.300Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.290356733Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175662024-01-10T18:02:28.300Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.29084047Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175672024-01-10T18:02:28.300Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.290923401Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175682024-01-10T18:02:28.300Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.290967624Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175692024-01-10T18:02:28.300Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291129913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175702024-01-10T18:02:28.300Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291243419Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175712024-01-10T18:02:28.300Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291323188Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175722024-01-10T18:02:28.300Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291351103Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175732024-01-10T18:02:28.300Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291373815Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175742024-01-10T18:02:28.300Z{"msg":"[0]R flush_numbers[0..12]: [285, 293, 293, 281, 292, 291, 285, 293, 292, 292, 290, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291397297Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175752024-01-10T18:02:28.301Z{"msg":"[0]R generation[0..12]: [160, 163, 163, 158, 162, 162, 160, 163, 162, 162, 162, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291420593Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175762024-01-10T18:02:28.301Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291444084Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175772024-01-10T18:02:28.301Z{"msg":"[1]R flush_numbers[0..12]: [285, 293, 293, 281, 292, 291, 285, 293, 292, 292, 290, 277]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291493437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175782024-01-10T18:02:28.302Z{"msg":"[1]R generation[0..12]: [160, 163, 163, 158, 162, 162, 160, 163, 162, 162, 162, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291518255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175792024-01-10T18:02:28.302Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291541563Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175802024-01-10T18:02:28.302Z{"msg":"[2]R flush_numbers[0..12]: [285, 293, 293, 281, 292, 291, 285, 293, 292, 292, 290, 277]"Wait for a query_work_queue command to finish before sending IO
175812024-01-10T18:02:28.302Z,"v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291595922Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175822024-01-10T18:02:28.302Z{"msg":"[2]R generation[0..12]: [160, 163, 163, 158, 162, 162, 160, 163, 162, 162, 162, 156]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291623277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175832024-01-10T18:02:28.302Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291646736Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175842024-01-10T18:02:28.302Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291688057Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175852024-01-10T18:02:28.302Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291711483Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175862024-01-10T18:02:28.302Z{"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291732374Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175872024-01-10T18:02:28.302Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291770947Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175882024-01-10T18:02:28.303Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291793774Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175892024-01-10T18:02:28.303Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291813368Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175902024-01-10T18:02:28.303Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291834703Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175912024-01-10T18:02:28.303Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291856403Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175922024-01-10T18:02:28.303Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291896427Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175932024-01-10T18:02:28.303Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291917742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175942024-01-10T18:02:28.303Z{"msg":"ebeb499c-2929-40c9-bf6f-4d883cfa0121 is now active with session: 00216365-f3ad-41b5-b796-c9670b6b1f94","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291938874Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175952024-01-10T18:02:28.303Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291959314Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814,"session_id":"00216365-f3ad-41b5-b796-c9670b6b1f94"}
175962024-01-10T18:02:28.303Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:28.291979613Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1814}
175972024-01-10T18:02:28.303ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
175982024-01-10T18:02:28.303ZLoading write count information from file "/tmp/test_repair_verify.data"
175992024-01-10T18:02:28.304ZRun Repair workload
176002024-01-10T18:02:28.304Z01/30 Write block 225 len 24576 data: 10 10 10 10 10 8
176012024-01-10T18:02:28.304Z02/30 Read block 44 len 12288
176022024-01-10T18:02:28.310Z03/30 Read block 19 len 36864
176032024-01-10T18:02:28.310Z04/30 Read block 508 len 24576
176042024-01-10T18:02:28.310Z05/30 Read block 290 len 4096
176052024-01-10T18:02:28.310Z06/30 Write block 567 len 24576 data: 10 11 11 10 11 10
176062024-01-10T18:02:28.311Z07/30 Flush
176072024-01-10T18:02:28.311Z08/30 Write block 582 len 12288 data: 14 14 11
176082024-01-10T18:02:28.315Z09/30 Write block 142 len 12288 data: 14 12 12
176092024-01-10T18:02:28.315Z10/30 Write block 278 len 8192 data: 12 12
176102024-01-10T18:02:28.315Z11/30 Read block 22 len 20480
176112024-01-10T18:02:28.315Z12/30 Flush
176122024-01-10T18:02:28.319Z13/30 Write block 146 len 24576 data: 13 14 13 13 12 11
176132024-01-10T18:02:28.319Z14/30 Flush
176142024-01-10T18:02:28.323Z15/30 Write block 277 len 8192 data: 13 13
176152024-01-10T18:02:28.323Z16/30 Read block 562 len 20480
176162024-01-10T18:02:28.323Z17/30 Read block 110 len 32768
176172024-01-10T18:02:28.327Z18/30 Read block 260 len 24576
176182024-01-10T18:02:28.330Z19/30 Write block 37 len 24576 data: 13 10 8 7 10 11
176192024-01-10T18:02:28.330Z20/30 Read block 406 len 36864
176202024-01-10T18:02:28.354Z21/30 Read block 57 len 32768
176212024-01-10T18:02:28.358Z22/30 Read block 525 len 8192
176222024-01-10T18:02:28.358Z23/30 Read block 37 len 8192
176232024-01-10T18:02:28.359Z24/30 Write block 539 len 28672 data: 14 17 19 11 11 12 13
176242024-01-10T18:02:28.359Z25/30 Read block 367 len 28672
176252024-01-10T18:02:28.359Z26/30 Write block 563 len 24576 data: 12 11 12 8 11 12
176262024-01-10T18:02:28.359Z27/30 Read block 113 len 28672
176272024-01-10T18:02:28.359Z28/30 Read block 200 len 20480
176282024-01-10T18:02:28.359Z29/30 Write block 409 len 32768 data: 13 13 14 11 11 10 8 9
176292024-01-10T18:02:28.359Z30/30 Read block 579 len 8192
176302024-01-10T18:02:28.362Z----------------------------------------------------------------
176312024-01-10T18:02:28.362Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
176322024-01-10T18:02:28.362ZStates: Active Active Active
176332024-01-10T18:02:28.362ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
176342024-01-10T18:02:28.363Z 1 Acked 1000 Write 6 Done Done Done false
176352024-01-10T18:02:28.363Z 2 Acked 1001 Read 3 Done Done Done false
176362024-01-10T18:02:28.363Z 3 Acked 1002 Read 9 Done Sent Done false
176372024-01-10T18:02:28.363Z 4 Acked 1003 Read 6 Done Sent Done false
176382024-01-10T18:02:28.363Z 5 Acked 1004 Read 1 Done Sent Done false
176392024-01-10T18:02:28.363Z 6 Acked 1005 Write 6 Done Sent Done false
176402024-01-10T18:02:28.363Z 7 Acked 1006 Flush 0 Done Sent Done false
176412024-01-10T18:02:28.363Z 8 Acked 1007 Write 3 Done Sent Done false
176422024-01-10T18:02:28.363Z 9 Acked 1008 Write 3 Done Sent Done false
176432024-01-10T18:02:28.364Z 10 Acked 1009 Write 2 Done Sent Done false
176442024-01-10T18:02:28.364Z 11 Acked 1010 Read 5 Done Sent Done false
176452024-01-10T18:02:28.364Z 12 Acked 1011 Flush 0 Done Sent Done false
176462024-01-10T18:02:28.364Z 13 Acked 1012 Write 6 Done Sent Done false
176472024-01-10T18:02:28.364Z 14 Acked 1013 Flush 0 Done Sent Done false
176482024-01-10T18:02:28.364Z 15 Acked 1014 Write 2 Done Sent Done false
176492024-01-10T18:02:28.364Z 16 Acked 1015 Read 5 Done Sent Done false
176502024-01-10T18:02:28.364Z 17 Acked 1016 Read 8 Done Sent Done false
176512024-01-10T18:02:28.367Z 18 Acked 1017 Read 6 Done Sent Done false
176522024-01-10T18:02:28.367Z 19 Acked 1018 Write 6 Done Sent Done false
176532024-01-10T18:02:28.368Z 20 Acked 1019 Read 9 Done Sent Done false
176542024-01-10T18:02:28.368Z 21 Acked 1020 Read 8 Done Sent Done false
176552024-01-10T18:02:28.368Z 22 Acked 1021 Read 2 Done Sent Done false
176562024-01-10T18:02:28.368Z 23 Acked 1022 Read 2 Done Sent Done false
176572024-01-10T18:02:28.368Z 24 Acked 1023 Write 7 Done Sent Done false
176582024-01-10T18:02:28.368Z 25 Acked 1024 Read 7 Done Sent Done false
176592024-01-10T18:02:28.368Z 26 Acked 1025 Write 6 Done Sent Done false
176602024-01-10T18:02:28.368Z 27 Acked 1026 Read 7 Done Sent Done false
176612024-01-10T18:02:28.369Z 28 Acked 1027 Read 5 Done Sent Done false
176622024-01-10T18:02:28.369Z 29 Acked 1028 Write 8 Done Sent Done false
176632024-01-10T18:02:28.369Z 30 Acked 1029 Read 2 Done Sent Done false
176642024-01-10T18:02:28.369Z STATES DS:0 DS:1 DS:2 TOTAL
176652024-01-10T18:02:28.369Z New 0 0 0 0
176662024-01-10T18:02:28.369Z Sent 0 28 0 28
176672024-01-10T18:02:28.369Z Done 30 2 30 62
176682024-01-10T18:02:28.369Z Skipped 0 0 0 0
176692024-01-10T18:02:28.369Z Error 0 0 0 0
176702024-01-10T18:02:28.369ZLast Flush: 1013 0 1013
176712024-01-10T18:02:28.369ZDownstairs last five completed:
176722024-01-10T18:02:28.369ZUpstairs last five completed: 30 29 28 27 26
176732024-01-10T18:02:28.369ZWrote out file "/tmp/test_repair_verify.data"
176742024-01-10T18:02:28.375Z
176752024-01-10T18:02:28.375ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
176762024-01-10T18:02:28.390ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176772024-01-10T18:02:28.390Z 0 000-019 160 160 160 285 285 285 F F F
176782024-01-10T18:02:28.390Z 1 020-039 163 163 163 293 293 293 T F T <---
176792024-01-10T18:02:28.396Z 2 040-059 163 163 163 293 293 293 T F T <---
176802024-01-10T18:02:28.396Z 3 060-079 158 158 158 281 281 281 F F F
176812024-01-10T18:02:28.396Z 4 080-099 162 162 162 292 292 292 F F F
176822024-01-10T18:02:28.396Z 5 100-119 162 162 162 291 291 291 F F F
176832024-01-10T18:02:28.396Z 6 120-139 160 160 160 285 285 285 F F F
176842024-01-10T18:02:28.397Z 7 140-159 164 163 164 296 293 296 F F F <---
176852024-01-10T18:02:28.397Z 8 160-179 162 162 162 292 292 292 F F F
176862024-01-10T18:02:28.397Z 9 180-199 162 162 162 292 292 292 F F F
176872024-01-10T18:02:28.397Z 10 200-219 162 162 162 290 290 290 F F F
176882024-01-10T18:02:28.397Z 11 220-239 164 156 164 294 277 294 F T F <---
176892024-01-10T18:02:28.397Z 12 240-259 163 163 163 293 293 293 F F F
176902024-01-10T18:02:28.397Z 13 260-279 164 163 164 295 293 295 T F T <---
176912024-01-10T18:02:28.397Z 14 280-299 162 162 162 290 290 290 F F F
176922024-01-10T18:02:28.397Z 15 300-319 160 160 160 285 285 285 F F F
176932024-01-10T18:02:28.397Z 16 320-339 155 155 155 274 274 274 F F F
176942024-01-10T18:02:28.397Z 17 340-359 161 161 161 288 288 288 F F F
176952024-01-10T18:02:28.397Z 18 360-379 162 162 162 292 292 292 F F F
176962024-01-10T18:02:28.397Z 19 380-399 160 160 160 284 284 284 F F F
176972024-01-10T18:02:28.398Z 20 400-419 162 162 162 292 292 292 T F T <---
176982024-01-10T18:02:28.398Z 21 420-439 159 159 159 283 283 283 F F F
176992024-01-10T18:02:28.398Z 22 440-459 159 159 159 283 283 283 F F F
177002024-01-10T18:02:28.398Z 23 460-479 162 162 162 291 291 291 F F F
177012024-01-10T18:02:28.398Z 24 480-499 162 162 162 292 292 292 F F F
177022024-01-10T18:02:28.398Z 25 500-519 160 160 160 284 284 284 F F F
177032024-01-10T18:02:28.398Z 26 520-539 159 159 159 283 283 283 T F T <---
177042024-01-10T18:02:28.398Z 27 540-559 157 157 157 278 278 278 T F T <---
177052024-01-10T18:02:28.398Z 28 560-579 164 162 164 294 290 294 T F T <---
177062024-01-10T18:02:28.398Z 29 580-599 164 160 164 295 286 295 F F F <---
177072024-01-10T18:02:28.399ZMax gen: 164, Max flush: 296
177082024-01-10T18:02:28.399ZError: Difference in extent metadata found!
177092024-01-10T18:02:28.399ZA Difference in extent metadata is expected here
177102024-01-10T18:02:28.399ZOn loop 82
177112024-01-10T18:02:28.399Z
177122024-01-10T18:02:28.413ZVerifying data now
177132024-01-10T18:02:30.752ZNo change after verify
177142024-01-10T18:02:30.752ZLoop: 82 Downstairs dump after verify (and repair):
177152024-01-10T18:02:31.077ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177162024-01-10T18:02:31.077Z 0 000-019 160 160 160 285 285 285 F F F
177172024-01-10T18:02:31.077Z 1 020-039 165 165 165 297 297 297 F F F
177182024-01-10T18:02:31.077Z 2 040-059 165 165 165 297 297 297 F F F
177192024-01-10T18:02:31.078Z 3 060-079 158 158 158 281 281 281 F F F
177202024-01-10T18:02:31.078Z 4 080-099 162 162 162 292 292 292 F F F
177212024-01-10T18:02:31.078Z 5 100-119 162 162 162 291 291 291 F F F
177222024-01-10T18:02:31.078Z 6 120-139 160 160 160 285 285 285 F F F
177232024-01-10T18:02:31.078Z 7 140-159 164 164 164 296 296 296 F F F
177242024-01-10T18:02:31.078Z 8 160-179 162 162 162 292 292 292 F F F
177252024-01-10T18:02:31.078Z 9 180-199 162 162 162 292 292 292 F F F
177262024-01-10T18:02:31.078Z 10 200-219 162 162 162 290 290 290 F F F
177272024-01-10T18:02:31.078Z 11 220-239 164 164 164 294 294 294 F F F
177282024-01-10T18:02:31.078Z 12 240-259 163 163 163 293 293 293 F F F
177292024-01-10T18:02:31.078Z 13 260-279 165 165 165 297 297 297 F F F
177302024-01-10T18:02:31.078Z 14 280-299 162 162 162 290 290 290 F F F
177312024-01-10T18:02:31.078Z 15 300-319 160 160 160 285 285 285 F F F
177322024-01-10T18:02:31.148Z 16 320-339 155 155 155 274 274 274 F F F
177332024-01-10T18:02:31.148Z 17 340-359 161 161 161 288 288 288 F F F
177342024-01-10T18:02:31.149Z 18 360-379 162 162 162 292 292 292 F F F
177352024-01-10T18:02:31.149Z 19 380-399 160 160 160 284 284 284 F F F
177362024-01-10T18:02:31.149Z 20 400-419 165 165 165 297 297 297 F F F
177372024-01-10T18:02:31.149Z 21 420-439 159 159 159 283 283 283 F F F
177382024-01-10T18:02:31.149Z 22 440-459 159 159 159 283 283 283 F F F
177392024-01-10T18:02:31.149Z 23 460-479 162 162 162 291 291 291 F F F
177402024-01-10T18:02:31.149Z 24 480-499 162 162 162 292 292 292 F F F
177412024-01-10T18:02:31.149Z 25 500-519 160 160 160 284 284 284 F F F
177422024-01-10T18:02:31.149Z 26 520-539 165 165 165 297 297 297 F F F
177432024-01-10T18:02:31.149Z 27 540-559 165 165 165 297 297 297 F F F
177442024-01-10T18:02:31.149Z 28 560-579 165 165 165 297 297 297 F F F
177452024-01-10T18:02:31.149Z 29 580-599 164 164 164 295 295 295 F F F
177462024-01-10T18:02:31.149ZMax gen: 165, Max flush: 297
177472024-01-10T18:02:31.149Z
177482024-01-10T18:02:31.149ZBegin loop 83
177492024-01-10T18:02:31.149ZDownstairs to restart: 0
177502024-01-10T18:02:31.149Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.497608354Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822}
177512024-01-10T18:02:31.149Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.498153354Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822}
177522024-01-10T18:02:31.150Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.498183868Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822}
177532024-01-10T18:02:31.150Z{"msg":"Crucible 9c3a1e1e-f7d9-4267-8ac9-2920ef793057 has session id: b3c84af2-b35e-4df5-bcf8-88cd145cce16","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.498241452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177542024-01-10T18:02:31.150Z{"msg":"Upstairs opts: Upstairs UUID: 9c3a1e1e-f7d9-4267-8ac9-2920ef793057, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.498333272Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177552024-01-10T18:02:31.150Z{Crucible runtime is spawned
177562024-01-10T18:02:31.150Z"msg":"Crucible stats registered with UUID: 9c3a1e1e-f7d9-4267-8ac9-2920ef793057","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.498390727Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177572024-01-10T18:02:31.150Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.49852404Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822}
177582024-01-10T18:02:31.150Z{"msg":"9c3a1e1e-f7d9-4267-8ac9-2920ef793057 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.498752255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177592024-01-10T18:02:31.150Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.498822301Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177602024-01-10T18:02:31.150Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.498868482Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177612024-01-10T18:02:31.150Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.498946992Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177622024-01-10T18:02:31.150Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.49899285Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177632024-01-10T18:02:31.150Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.499059265Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177642024-01-10T18:02:31.150Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.499103215Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177652024-01-10T18:02:31.150Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.499166009Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177662024-01-10T18:02:31.151Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.499192046Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177672024-01-10T18:02:31.151Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.499315511Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177682024-01-10T18:02:31.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.499513789Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177692024-01-10T18:02:31.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.499569995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177702024-01-10T18:02:31.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.499614913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177712024-01-10T18:02:31.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.500825967Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177722024-01-10T18:02:31.151Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.500892497Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177732024-01-10T18:02:31.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.500917988Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177742024-01-10T18:02:31.151Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.500967984Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177752024-01-10T18:02:31.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.501748571Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177762024-01-10T18:02:31.151Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.501831132Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177772024-01-10T18:02:31.151Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.50187989Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177782024-01-10T18:02:31.151Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.502181057Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177792024-01-10T18:02:31.151Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.502820791Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177802024-01-10T18:02:31.151Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.502904134Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177812024-01-10T18:02:31.152Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.502951105Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177822024-01-10T18:02:31.152Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.50301023Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177832024-01-10T18:02:31.152Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.503036981Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177842024-01-10T18:02:31.162Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.50306399Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177852024-01-10T18:02:31.162Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.503117415Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177862024-01-10T18:02:31.162Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.503834013Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177872024-01-10T18:02:31.162Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.503900089Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177882024-01-10T18:02:31.162Z{"msg":"[0]R flush_numbers[0..12]: [285, 297, 297, 281, 292, 291, 285, 296, 292, 292, 290, 294]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.503928777Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177892024-01-10T18:02:31.162Z{"msg":"[0]R generation[0..12]: [160, 165, 165, 158, 162, 162, 160, 164, 162, 162, 162, 164]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.503972159Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177902024-01-10T18:02:31.162Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504030169Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177912024-01-10T18:02:31.162Z{"msg":"[1]R flush_numbers[0..12]: [285, 297, 297, 281, 292, 291, 285, 296, 292, 292, 290, 294]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504055898Z","hostname":"Wait for a query_work_queue command to finish before sending IO
177922024-01-10T18:02:31.163Zip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177932024-01-10T18:02:31.163Z{"msg":"[1]R generation[0..12]: [160, 165, 165, 158, 162, 162, 160, 164, 162, 162, 162, 164]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504103571Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177942024-01-10T18:02:31.235Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.50412893Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177952024-01-10T18:02:31.235Z{"msg":"[2]R flush_numbers[0..12]: [285, 297, 297, 281, 292, 291, 285, 296, 292, 292, 290, 294]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504152628Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177962024-01-10T18:02:31.235Z{"msg":"[2]R generation[0..12]: [160, 165, 165, 158, 162, 162, 160, 164, 162, 162, 162, 164]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504209662Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177972024-01-10T18:02:31.235Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504268653Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177982024-01-10T18:02:31.235Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504302171Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
177992024-01-10T18:02:31.235Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504370302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178002024-01-10T18:02:31.235Z{"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504399189Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178012024-01-10T18:02:31.235Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.50445579Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178022024-01-10T18:02:31.235Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504493102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178032024-01-10T18:02:31.235Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504514767Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178042024-01-10T18:02:31.235Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504538097Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178052024-01-10T18:02:31.235Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504564668Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178062024-01-10T18:02:31.235Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504630421Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178072024-01-10T18:02:31.235Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504668409Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178082024-01-10T18:02:31.235Z{"msg":"9c3a1e1e-f7d9-4267-8ac9-2920ef793057 is now active with session: b3c84af2-b35e-4df5-bcf8-88cd145cce16","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504707233Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178092024-01-10T18:02:31.236Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:29.504739453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822,"session_id":"b3c84af2-b35e-4df5-bcf8-88cd145cce16"}
178102024-01-10T18:02:31.236Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178112024-01-10T18:02:31.236Zlevel":30,"time":"2024-01-10T18:02:29.504844652Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1822}
178122024-01-10T18:02:31.236ZLoading write count information from file "/tmp/test_repair_verify.data"
178132024-01-10T18:02:31.236ZRun Repair workload
178142024-01-10T18:02:31.236Z01/30 Flush
178152024-01-10T18:02:31.847Z02/30 Write block 424 len 28672 data: 15 14 13 12 13 12 10
178162024-01-10T18:02:31.847Z03/30 Write block 281 len 28672 data: 12 10 11 13 15 13 14
178172024-01-10T18:02:31.847Z04/30 Flush
178182024-01-10T18:02:31.847Z05/30 Write block 249 len 16384 data: 12 10 9 10
178192024-01-10T18:02:31.847Z06/30 Write block 8 len 12288 data: 8 10 11
178202024-01-10T18:02:31.848Z07/30 Read block 490 len 4096
178212024-01-10T18:02:31.848Z08/30 Write block 405 len 40960 data: 19 20 20 16 14 14 15 12 12 11
178222024-01-10T18:02:31.848Z09/30 Read block 372 len 28672
178232024-01-10T18:02:31.848Z10/30 Write block 395 len 28672 data: 10 10 9 8 11 9 11
178242024-01-10T18:02:31.848Z11/30 Write block 294 len 8192 data: 14 15
178252024-01-10T18:02:31.848Z12/30 Write block 241 len 28672 data: 12 13 15 19 17 12 10
178262024-01-10T18:02:31.848Z13/30 Flush
178272024-01-10T18:02:31.848Z14/30 Write block 172 len 12288 data: 10 10 11
178282024-01-10T18:02:31.848Z15/30 Read block 40 len 16384
178292024-01-10T18:02:31.848Z16/30 Read block 576 len 28672
178302024-01-10T18:02:31.848Z17/30 Write block 166 len 28672 data: 19 19 17 16 13 11 11
178312024-01-10T18:02:31.848Z18/30 Read block 395 len 32768
178322024-01-10T18:02:31.848Z19/30 Flush
178332024-01-10T18:02:31.849Z20/30 Write block 422 len 24576 data: 17 16 16 15 14 13
178342024-01-10T18:02:31.849Z21/30 Read block 303 len 36864
178352024-01-10T18:02:31.849Z22/30 Write block 50 len 36864 data: 14 12 12 9 14 13 12 11 16
178362024-01-10T18:02:31.849Z23/30 Flush
178372024-01-10T18:02:31.849Z24/30 Write block 211 len 4096 data: 12
178382024-01-10T18:02:31.849Z25/30 Write block 355 len 8192 data: 8 10
178392024-01-10T18:02:31.849Z26/30 Read block 66 len 36864
178402024-01-10T18:02:31.849Z27/30 Read block 26 len 12288
178412024-01-10T18:02:31.849Z28/30 Write block 413 len 40960 data: 13 12 9 10 12 12 18 21 19 18
178422024-01-10T18:02:31.849Z29/30 Write block 528 len 24576 data: 12 13 14 18 15 16
178432024-01-10T18:02:31.849Z30/30 Write block 22 len 40960 data: 12 14 11 12 12 11 12 13 16 13
178442024-01-10T18:02:31.849Z----------------------------------------------------------------
178452024-01-10T18:02:31.849Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:29
178462024-01-10T18:02:31.850ZStates: Active Active Active
178472024-01-10T18:02:31.850ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
178482024-01-10T18:02:31.850Z 2 Acked 1001 Write 7 Done Done Done false
178492024-01-10T18:02:31.850Z 3 Acked 1002 Write 7 Sent Done Done false
178502024-01-10T18:02:31.850Z 4 Acked 1003 Flush 0 Sent Done Done false
178512024-01-10T18:02:31.850Z 5 Acked 1004 Write 4 Sent Done Done false
178522024-01-10T18:02:31.850Z 6 Acked 1005 Write 3 Sent Done Done false
178532024-01-10T18:02:31.850Z 7 Acked 1006 Read 1 Sent Done Done false
178542024-01-10T18:02:31.850Z 8 Acked 1007 Write 10 Sent Done Done false
178552024-01-10T18:02:31.850Z 9 Acked 1008 Read 7 Sent Done Done false
178562024-01-10T18:02:31.850Z 10 Acked 1009 Write 7 Sent Done Done false
178572024-01-10T18:02:31.850Z 11 Acked 1010 Write 2 Sent Done Done false
178582024-01-10T18:02:31.850Z 12 Acked 1011 Write 7 Sent Done Done false
178592024-01-10T18:02:31.850Z 13 Acked 1012 Flush 0 Sent Done Done false
178602024-01-10T18:02:31.850Z 14 Acked 1013 Write 3 Sent Done Done false
178612024-01-10T18:02:31.850Z 15 Acked 1014 Read 4 Sent Done Done false
178622024-01-10T18:02:31.850Z 16 Acked 1015 Read 7 Sent Done Done false
178632024-01-10T18:02:31.850Z 17 Acked 1016 Write 7 Sent Done Done false
178642024-01-10T18:02:31.850Z 18 Acked 1017 Read 8 Sent Done Done false
178652024-01-10T18:02:31.851Z 19 Acked 1018 Flush 0 Sent Done Done false
178662024-01-10T18:02:31.851Z 20 Acked 1019 Write 6 Sent Done Done false
178672024-01-10T18:02:31.851Z 21 Acked 1020 Read 9 Sent Done Done false
178682024-01-10T18:02:31.851Z 22 Acked 1021 Write 9 Sent Done Done false
178692024-01-10T18:02:31.851Z 23 Acked 1022 Flush 0 Sent Done Done false
178702024-01-10T18:02:31.851Z 24 Acked 1023 Write 1 Sent Done Done false
178712024-01-10T18:02:31.851Z 25 Acked 1024 Write 2 Sent Done Done false
178722024-01-10T18:02:31.851Z 26 Acked 1025 Read 9 Sent Done Done false
178732024-01-10T18:02:31.851Z 27 Acked 1026 Read 3 Sent Done Done false
178742024-01-10T18:02:31.851Z 28 Acked 1027 Write 10 Sent Done Done false
178752024-01-10T18:02:31.851Z 29 Acked 1028 Write 6 Sent Done Done false
178762024-01-10T18:02:31.851Z 30 Acked 1029 Write 10 Sent Sent Sent false
178772024-01-10T18:02:31.851Z STATES DS:0 DS:1 DS:2 TOTAL
178782024-01-10T18:02:31.851Z New 0 0 0 0
178792024-01-10T18:02:31.851Z Sent 28 1 1 30
178802024-01-10T18:02:31.851Z Done 1 28 28 57
178812024-01-10T18:02:31.851Z Skipped 0 0 0 0
178822024-01-10T18:02:31.851Z Error 0 0 0 0
178832024-01-10T18:02:31.851ZLast Flush: 1000 1022 1022
178842024-01-10T18:02:31.852ZDownstairs last five completed: 1000
178852024-01-10T18:02:31.852ZUpstairs last five completed: 30 29 28 27 26
178862024-01-10T18:02:31.916ZWrote out file "/tmp/test_repair_verify.data"
178872024-01-10T18:02:31.916Z
178882024-01-10T18:02:31.916ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
178892024-01-10T18:02:31.917ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178902024-01-10T18:02:31.917Z 0 000-019 160 166 166 285 300 300 F F F <---
178912024-01-10T18:02:31.917Z 1 020-039 165 165 165 297 297 297 F T T <---
178922024-01-10T18:02:31.917Z 2 040-059 165 166 166 297 302 302 F F F <---
178932024-01-10T18:02:31.917Z 3 060-079 158 158 158 281 281 281 F F F
178942024-01-10T18:02:31.917Z 4 080-099 162 162 162 292 292 292 F F F
178952024-01-10T18:02:31.917Z 5 100-119 162 162 162 291 291 291 F F F
178962024-01-10T18:02:31.917Z 6 120-139 160 160 160 285 285 285 F F F
178972024-01-10T18:02:31.917Z 7 140-159 164 164 164 296 296 296 F F F
178982024-01-10T18:02:31.917Z 8 160-179 162 166 166 292 301 301 F F F <---
178992024-01-10T18:02:31.917Z 9 180-199 162 162 162 292 292 292 F F F
179002024-01-10T18:02:31.917Z 10 200-219 162 162 162 290 290 290 F T T <---
179012024-01-10T18:02:31.917Z 11 220-239 164 164 164 294 294 294 F F F
179022024-01-10T18:02:31.917Z 12 240-259 163 166 166 293 300 300 F F F <---
179032024-01-10T18:02:31.917Z 13 260-279 165 165 165 297 297 297 F F F
179042024-01-10T18:02:31.917Z 14 280-299 162 166 166 290 300 300 F F F <---
179052024-01-10T18:02:31.917Z 15 300-319 160 160 160 285 285 285 F F F
179062024-01-10T18:02:31.917Z 16 320-339 155 155 155 274 274 274 F F F
179072024-01-10T18:02:31.918Z 17 340-359 161 161 161 288 288 288 F T T <---
179082024-01-10T18:02:31.918Z 18 360-379 162 162 162 292 292 292 F F F
179092024-01-10T18:02:31.953Z 19 380-399 160 166 166 284 300 300 F F F <---
179102024-01-10T18:02:31.953Z 20 400-419 165 166 166 297 300 300 F T T <---
179112024-01-10T18:02:31.953Z 21 420-439 159 166 166 283 302 302 T T T <---
179122024-01-10T18:02:31.953Z 22 440-459 159 159 159 283 283 283 F F F
179132024-01-10T18:02:31.953Z 23 460-479 162 162 162 291 291 291 F F F
179142024-01-10T18:02:31.953Z 24 480-499 162 162 162 292 292 292 F F F
179152024-01-10T18:02:32.406Z 25 500-519 160 160 160 284 284 284 F F F
179162024-01-10T18:02:32.406Z 26 520-539 165 165 165 297 297 297 F T T <---
179172024-01-10T18:02:32.406Z 27 540-559 165 165 165 297 297 297 F F F
179182024-01-10T18:02:32.406Z 28 560-579 165 165 165 297 297 297 F F F
179192024-01-10T18:02:32.406Z 29 580-599 164 164 164 295 295 295 F F F
179202024-01-10T18:02:32.406ZMax gen: 166, Max flush: 302
179212024-01-10T18:02:32.406ZError: Difference in extent metadata found!
179222024-01-10T18:02:32.406ZA Difference in extent metadata is expected here
179232024-01-10T18:02:32.406ZOn loop 83
179242024-01-10T18:02:32.407Z
179252024-01-10T18:02:32.407ZVerifying data now
179262024-01-10T18:02:34.961ZNo change after verify
179272024-01-10T18:02:34.961ZLoop: 83 Downstairs dump after verify (and repair):
179282024-01-10T18:02:35.257ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179292024-01-10T18:02:35.257Z 0 000-019 166 166 166 300 300 300 F F F
179302024-01-10T18:02:35.257Z 1 020-039 167 167 167 303 303 303 F F F
179312024-01-10T18:02:35.257Z 2 040-059 166 166 166 302 302 302 F F F
179322024-01-10T18:02:35.257Z 3 060-079 158 158 158 281 281 281 F F F
179332024-01-10T18:02:35.257Z 4 080-099 162 162 162 292 292 292 F F F
179342024-01-10T18:02:35.257Z 5 100-119 162 162 162 291 291 291 F F F
179352024-01-10T18:02:35.257Z 6 120-139 160 160 160 285 285 285 F F F
179362024-01-10T18:02:35.258Z 7 140-159 164 164 164 296 296 296 F F F
179372024-01-10T18:02:35.289Z 8 160-179 166 166 166 301 301 301 F F F
179382024-01-10T18:02:35.289Z 9 180-199 162 162 162 292 292 292 F F F
179392024-01-10T18:02:35.289Z 10 200-219 167 167 167 303 303 303 F F F
179402024-01-10T18:02:35.289Z 11 220-239 164 164 164 294 294 294 F F F
179412024-01-10T18:02:35.289Z 12 240-259 166 166 166 300 300 300 F F F
179422024-01-10T18:02:35.289Z 13 260-279 165 165 165 297 297 297 F F F
179432024-01-10T18:02:35.290Z 14 280-299 166 166 166 300 300 300 F F F
179442024-01-10T18:02:35.290Z 15 300-319 160 160 160 285 285 285 F F F
179452024-01-10T18:02:35.290Z 16 320-339 155 155 155 274 274 274 F F F
179462024-01-10T18:02:35.290Z 17 340-359 167 167 167 303 303 303 F F F
179472024-01-10T18:02:35.290Z 18 360-379 162 162 162 292 292 292 F F F
179482024-01-10T18:02:35.290Z 19 380-399 166 166 166 300 300 300 F F F
179492024-01-10T18:02:35.290Z 20 400-419 167 167 167 303 303 303 F F F
179502024-01-10T18:02:35.290Z 21 420-439 167 167 167 303 303 303 F F F
179512024-01-10T18:02:35.290Z 22 440-459 159 159 159 283 283 283 F F F
179522024-01-10T18:02:35.290Z 23 460-479 162 162 162 291 291 291 F F F
179532024-01-10T18:02:35.290Z 24 480-499 162 162 162 292 292 292 F F F
179542024-01-10T18:02:35.290Z 25 500-519 160 160 160 284 284 284 F F F
179552024-01-10T18:02:35.290Z 26 520-539 167 167 167 303 303 303 F F F
179562024-01-10T18:02:35.290Z 27 540-559 165 165 165 297 297 297 F F F
179572024-01-10T18:02:35.290Z 28 560-579 165 165 165 297 297 297 F F F
179582024-01-10T18:02:35.290Z 29 580-599 164 164 164 295 295 295 F F F
179592024-01-10T18:02:35.290ZMax gen: 167, Max flush: 303
179602024-01-10T18:02:35.290Z
179612024-01-10T18:02:35.291ZBegin loop 84
179622024-01-10T18:02:35.311ZDownstairs to restart: 1
179632024-01-10T18:02:35.311Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.867723154Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830}
179642024-01-10T18:02:35.311Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.868296292Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830}
179652024-01-10T18:02:35.311Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":Crucible runtime is spawned
179662024-01-10T18:02:35.311Z30,"time":"2024-01-10T18:02:34.868520862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830}
179672024-01-10T18:02:35.312Z{"msg":"Crucible 295b24ce-be6b-4d41-b90c-806e8fe3a0ad has session id: 69e432d2-c695-46dd-a2ac-4b3d832bcc36","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.86860263Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179682024-01-10T18:02:35.312Z{"msg":"Upstairs opts: Upstairs UUID: 295b24ce-be6b-4d41-b90c-806e8fe3a0ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.868690077Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179692024-01-10T18:02:35.312Z{"msg":"Crucible stats registered with UUID: 295b24ce-be6b-4d41-b90c-806e8fe3a0ad","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.868737743Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179702024-01-10T18:02:35.312Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.868806175Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830}
179712024-01-10T18:02:35.312Z{"msg":"295b24ce-be6b-4d41-b90c-806e8fe3a0ad active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.868870586Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179722024-01-10T18:02:35.312Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.86896658Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179732024-01-10T18:02:35.312Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.869071446Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179742024-01-10T18:02:35.312Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.869117222Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179752024-01-10T18:02:35.312Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.869204369Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179762024-01-10T18:02:35.312Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.869247776Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179772024-01-10T18:02:35.313Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.869377694Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179782024-01-10T18:02:35.313Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.869425351Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179792024-01-10T18:02:35.313Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.86955555Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179802024-01-10T18:02:35.313Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.86961398Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179812024-01-10T18:02:35.313Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.869662477Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179822024-01-10T18:02:35.313Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.869737126Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179832024-01-10T18:02:35.323Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.869808397Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179842024-01-10T18:02:35.323Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.870888107Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179852024-01-10T18:02:35.323Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.870964124Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179862024-01-10T18:02:35.323Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.871118787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179872024-01-10T18:02:35.323Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.871171406Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179882024-01-10T18:02:35.323Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.87142417Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179892024-01-10T18:02:35.324Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.871480343Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179902024-01-10T18:02:35.324Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.87207998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179912024-01-10T18:02:35.324Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.87228163Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179922024-01-10T18:02:35.324Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.872701331Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179932024-01-10T18:02:35.324Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873076972Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179942024-01-10T18:02:35.324Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873172544Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179952024-01-10T18:02:35.324Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873235766Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179962024-01-10T18:02:35.324Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873260015Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179972024-01-10T18:02:35.324Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873282752Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179982024-01-10T18:02:35.324Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873341625Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
179992024-01-10T18:02:35.324Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873530464Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180002024-01-10T18:02:35.324Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873586639Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180012024-01-10T18:02:35.324Z{"msg":"[0]R flush_numbers[0..12]: [300, 303, 302, 281, 292, 291, 285, 296, 301, 292, 303, 294]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.87362996Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180022024-01-10T18:02:35.334Z{"msg":"[0]R generation[0..12]: [166, 167, 166, 158, 162, 162, 160, 164, 166, 162, 167, 164]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873681351Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180032024-01-10T18:02:35.346Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873707724Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180042024-01-10T18:02:35.346Z{"msg":"[1]R flush_numbers[0..12]: [300, 303, 302, 281, 292, 291, 285, 296, 301, 292, 303, 294]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873749353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180052024-01-10T18:02:35.346Z{"msg":"[1]R generation[0..12]: [166, 167, 166, 158, 162, 162, 160, 164, 166, 162, 167, 164]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873773865Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180062024-01-10T18:02:35.346Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873822706Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180072024-01-10T18:02:35.347Z{"msg":"[2]R flush_numbers[0..12]: [300, 303, 302, 281, 292, 291, 285, 296, 301, 292, 303, 294]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
180082024-01-10T18:02:35.347Z,"time":"2024-01-10T18:02:34.873847267Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180092024-01-10T18:02:35.347Z{"msg":"[2]R generation[0..12]: [166, 167, 166, 158, 162, 162, 160, 164, 166, 162, 167, 164]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.87395169Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180102024-01-10T18:02:35.347Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.873981317Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180112024-01-10T18:02:35.347Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874022304Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180122024-01-10T18:02:35.347Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874046081Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180132024-01-10T18:02:35.378Z{"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874067361Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180142024-01-10T18:02:35.379Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874114583Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180152024-01-10T18:02:35.379Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874137422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180162024-01-10T18:02:35.379Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874185535Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180172024-01-10T18:02:35.379Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874210506Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180182024-01-10T18:02:35.379Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874259258Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180192024-01-10T18:02:35.379Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874288977Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180202024-01-10T18:02:35.379Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874324795Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180212024-01-10T18:02:35.379Z{"msg":"295b24ce-be6b-4d41-b90c-806e8fe3a0ad is now active with session: 69e432d2-c695-46dd-a2ac-4b3d832bcc36","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874352256Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180222024-01-10T18:02:35.379Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874372264Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830,"session_id":"69e432d2-c695-46dd-a2ac-4b3d832bcc36"}
180232024-01-10T18:02:35.379Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:34.874392322Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1830}
180242024-01-10T18:02:35.472ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
180252024-01-10T18:02:35.472ZLoading write count information from file "/tmp/test_repair_verify.data"
180262024-01-10T18:02:35.472ZRun Repair workload
180272024-01-10T18:02:35.472Z01/30 Write block 16 len 32768 data: 12 12 14 13 12 13 13 15
180282024-01-10T18:02:35.947Z02/30 Read block 213 len 32768
180292024-01-10T18:02:35.947Z03/30 Read block 362 len 8192
180302024-01-10T18:02:35.947Z04/30 Write block 0 len 8192 data: 5 6
180312024-01-10T18:02:35.948Z05/30 Write block 463 len 32768 data: 11 11 9 12 13 14 11 11
180322024-01-10T18:02:35.948Z06/30 Read block 111 len 20480
180332024-01-10T18:02:35.948Z07/30 Write block 158 len 36864 data: 12 11 13 16 17 15 17 19 20
180342024-01-10T18:02:35.948Z08/30 Write block 560 len 28672 data: 10 11 12 13 12 13 9
180352024-01-10T18:02:35.948Z09/30 Flush
180362024-01-10T18:02:35.948Z10/30 Write block 413 len 12288 data: 14 13 10
180372024-01-10T18:02:35.949Z11/30 Read block 101 len 24576
180382024-01-10T18:02:35.949Z12/30 Write block 33 len 16384 data: 13 14 14 15
180392024-01-10T18:02:35.949Z13/30 Write block 233 len 20480 data: 5 7 8 8 10
180402024-01-10T18:02:35.949Z14/30 Read block 469 len 36864
180412024-01-10T18:02:35.949Z15/30 Read block 204 len 20480
180422024-01-10T18:02:35.949Z16/30 Write block 564 len 4096 data: 13
180432024-01-10T18:02:35.949Z17/30 Read block 558 len 24576
180442024-01-10T18:02:35.949Z18/30 Write block 69 len 8192 data: 10 11
180452024-01-10T18:02:35.949Z19/30 Flush
180462024-01-10T18:02:35.949Z20/30 Write block 63 len 32768 data: 12 13 15 17 16 13 11 12
180472024-01-10T18:02:35.993Z21/30 Read block 223 len 40960
180482024-01-10T18:02:35.993Z22/30 Read block 50 len 40960
180492024-01-10T18:02:35.993Z23/30 Write block 539 len 8192 data: 15 18
180502024-01-10T18:02:35.993Z24/30 Write block 174 len 28672 data: 12 9 8 10 10 11 14
180512024-01-10T18:02:35.993Z25/30 Flush
180522024-01-10T18:02:35.993Z26/30 Flush
180532024-01-10T18:02:35.993Z27/30 Write block 566 len 36864 data: 10 12 13 12 11 12 11 11 15
180542024-01-10T18:02:35.993Z28/30 Write block 317 len 4096 data: 10
180552024-01-10T18:02:35.993Z29/30 Read block 555 len 40960
180562024-01-10T18:02:35.993Z30/30 Read block 215 len 40960
180572024-01-10T18:02:35.994Z----------------------------------------------------------------
180582024-01-10T18:02:35.994Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
180592024-01-10T18:02:35.994ZStates: Active Active Active
180602024-01-10T18:02:35.994ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
180612024-01-10T18:02:35.994Z 1 Acked 1000 Write 8 Done Done Done false
180622024-01-10T18:02:35.994Z 2 Acked 1001 Read 8 Done Done Done false
180632024-01-10T18:02:35.994Z 3 Acked 1002 Read 2 Done Done Done false
180642024-01-10T18:02:35.994Z 4 Acked 1003 Write 2 Done Done Done false
180652024-01-10T18:02:35.994Z 5 Acked 1004 Write 8 Done Sent Done false
180662024-01-10T18:02:35.994Z 6 Acked 1005 Read 5 Done Sent Done false
180672024-01-10T18:02:35.994Z 7 Acked 1006 Write 9 Done Sent Done false
180682024-01-10T18:02:35.994Z 8 Acked 1007 Write 7 Done Sent Done false
180692024-01-10T18:02:35.994Z 9 Acked 1008 Flush 0 Done Sent Done false
180702024-01-10T18:02:35.994Z 10 Acked 1009 Write 3 Done Sent Done false
180712024-01-10T18:02:35.994Z 11 Acked 1010 Read 6 Done Sent Done false
180722024-01-10T18:02:35.994Z 12 Acked 1011 Write 4 Done Sent Done false
180732024-01-10T18:02:35.994Z 13 Acked 1012 Write 5 Done Sent Done false
180742024-01-10T18:02:35.994Z 14 Acked 1013 Read 9 Done Sent Done false
180752024-01-10T18:02:35.995Z 15 Acked 1014 Read 5 Done Sent Done false
180762024-01-10T18:02:35.995Z 16 Acked 1015 Write 1 Done Sent Done false
180772024-01-10T18:02:35.995Z 17 Acked 1016 Read 6 Done Sent Done false
180782024-01-10T18:02:35.995Z 18 Acked 1017 Write 2 Done Sent Done false
180792024-01-10T18:02:35.995Z 19 Acked 1018 Flush 0 Done Sent Done false
180802024-01-10T18:02:35.995Z 20 Acked 1019 Write 8 Done Sent Done false
180812024-01-10T18:02:35.995Z 21 Acked 1020 Read 10 Done Sent Done false
180822024-01-10T18:02:35.995Z 22 Acked 1021 Read 10 Done Sent Done false
180832024-01-10T18:02:35.995Z 23 Acked 1022 Write 2 Done Sent Done false
180842024-01-10T18:02:35.995Z 24 Acked 1023 Write 7 Done Sent Done false
180852024-01-10T18:02:35.995Z 25 Acked 1024 Flush 0 Done Sent Done false
180862024-01-10T18:02:35.995Z 26 Acked 1025 Flush 0 Done Sent Done false
180872024-01-10T18:02:35.995Z 27 Acked 1026 Write 9 Done Sent Done false
180882024-01-10T18:02:35.995Z 28 Acked 1027 Write 1 Done Sent Done false
180892024-01-10T18:02:35.995Z 29 Acked 1028 Read 10 Done Sent Done false
180902024-01-10T18:02:35.995Z 30 Acked 1029 Read 10 Done Sent Sent false
180912024-01-10T18:02:35.995Z STATES DS:0 DS:1 DS:2 TOTAL
180922024-01-10T18:02:35.996Z New 0 0 0 0
180932024-01-10T18:02:35.996Z Sent 0 26 1 27
180942024-01-10T18:02:36.046Z Done 30 4 29 63
180952024-01-10T18:02:36.174Z Skipped 0 0 0 0
180962024-01-10T18:02:36.174Z Error 0 0 0 0
180972024-01-10T18:02:36.174ZLast Flush: 1025 0 1025
180982024-01-10T18:02:36.174ZDownstairs last five completed:
180992024-01-10T18:02:36.174ZUpstairs last five completed: 30 29 28 27 26
181002024-01-10T18:02:36.174ZWrote out file "/tmp/test_repair_verify.data"
181012024-01-10T18:02:36.174Z
181022024-01-10T18:02:36.174ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181032024-01-10T18:02:36.174ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181042024-01-10T18:02:36.174Z 0 000-019 168 166 168 304 300 304 F T F <---
181052024-01-10T18:02:36.175Z 1 020-039 168 167 168 305 303 305 F T F <---
181062024-01-10T18:02:36.175Z 2 040-059 166 166 166 302 302 302 F F F
181072024-01-10T18:02:36.175Z 3 060-079 168 158 168 306 281 306 F F F <---
181082024-01-10T18:02:36.175Z 4 080-099 162 162 162 292 292 292 F F F
181092024-01-10T18:02:36.175Z 5 100-119 162 162 162 291 291 291 F F F
181102024-01-10T18:02:36.175Z 6 120-139 160 160 160 285 285 285 F F F
181112024-01-10T18:02:36.175Z 7 140-159 168 164 168 304 296 304 F F F <---
181122024-01-10T18:02:36.175Z 8 160-179 168 166 168 306 301 306 F F F <---
181132024-01-10T18:02:36.175Z 9 180-199 168 162 168 306 292 306 F F F <---
181142024-01-10T18:02:36.175Z 10 200-219 167 167 167 303 303 303 F F F
181152024-01-10T18:02:36.175Z 11 220-239 168 164 168 305 294 305 F F F <---
181162024-01-10T18:02:36.175Z 12 240-259 166 166 166 300 300 300 F F F
181172024-01-10T18:02:36.175Z 13 260-279 165 165 165 297 297 297 F F F
181182024-01-10T18:02:36.175Z 14 280-299 166 166 166 300 300 300 F F F
181192024-01-10T18:02:36.210Z 15 300-319 160 160 160 285 285 285 T F T <---
181202024-01-10T18:02:36.211Z 16 320-339 155 155 155 274 274 274 F F F
181212024-01-10T18:02:36.211Z 17 340-359 167 167 167 303 303 303 F F F
181222024-01-10T18:02:36.211Z 18 360-379 162 162 162 292 292 292 F F F
181232024-01-10T18:02:36.211Z 19 380-399 166 166 166 300 300 300 F F F
181242024-01-10T18:02:36.211Z 20 400-419 168 167 168 305 303 305 F F F <---
181252024-01-10T18:02:36.211Z 21 420-439 167 167 167 303 303 303 F F F
181262024-01-10T18:02:36.211Z 22 440-459 159 159 159 283 283 283 F F F
181272024-01-10T18:02:36.211Z 23 460-479 168 162 168 304 291 304 F F F <---
181282024-01-10T18:02:37.086Z 24 480-499 162 162 162 292 292 292 F F F
181292024-01-10T18:02:37.088Z 25 500-519 160 160 160 284 284 284 F F F
181302024-01-10T18:02:37.088Z 26 520-539 168 167 168 306 303 306 F F F <---
181312024-01-10T18:02:37.088Z 27 540-559 168 165 168 306 297 306 F F F <---
181322024-01-10T18:02:37.089Z 28 560-579 168 165 168 305 297 305 T F T <---
181332024-01-10T18:02:37.089Z 29 580-599 164 164 164 295 295 295 F F F
181342024-01-10T18:02:37.089ZMax gen: 168, Max flush: 306
181352024-01-10T18:02:37.089ZError: Difference in extent metadata found!
181362024-01-10T18:02:37.089ZA Difference in extent metadata is expected here
181372024-01-10T18:02:37.090ZOn loop 84
181382024-01-10T18:02:37.090Z
181392024-01-10T18:02:37.090ZVerifying data now
181402024-01-10T18:02:53.762ZNo change after verify
181412024-01-10T18:02:53.762ZLoop: 84 Downstairs dump after verify (and repair):
181422024-01-10T18:02:54.044ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181432024-01-10T18:02:54.045Z 0 000-019 168 168 168 304 304 304 F F F
181442024-01-10T18:02:54.045Z 1 020-039 168 168 168 305 305 305 F F F
181452024-01-10T18:02:54.046Z 2 040-059 166 166 166 302 302 302 F F F
181462024-01-10T18:02:54.046Z 3 060-079 168 168 168 306 306 306 F F F
181472024-01-10T18:02:54.046Z 4 080-099 162 162 162 292 292 292 F F F
181482024-01-10T18:02:54.046Z 5 100-119 162 162 162 291 291 291 F F F
181492024-01-10T18:02:54.046Z 6 120-139 160 160 160 285 285 285 F F F
181502024-01-10T18:02:54.047Z 7 140-159 168 168 168 304 304 304 F F F
181512024-01-10T18:02:54.047Z 8 160-179 168 168 168 306 306 306 F F F
181522024-01-10T18:02:54.047Z 9 180-199 168 168 168 306 306 306 F F F
181532024-01-10T18:02:54.047Z 10 200-219 167 167 167 303 303 303 F F F
181542024-01-10T18:02:54.047Z 11 220-239 168 168 168 305 305 305 F F F
181552024-01-10T18:02:54.047Z 12 240-259 166 166 166 300 300 300 F F F
181562024-01-10T18:02:54.047Z 13 260-279 165 165 165 297 297 297 F F F
181572024-01-10T18:02:54.047Z 14 280-299 166 166 166 300 300 300 F F F
181582024-01-10T18:02:54.047Z 15 300-319 169 169 169 307 307 307 F F F
181592024-01-10T18:02:54.047Z 16 320-339 155 155 155 274 274 274 F F F
181602024-01-10T18:02:54.047Z 17 340-359 167 167 167 303 303 303 F F F
181612024-01-10T18:02:54.048Z 18 360-379 162 162 162 292 292 292 F F F
181622024-01-10T18:02:54.048Z 19 380-399 166 166 166 300 300 300 F F F
181632024-01-10T18:02:54.048Z 20 400-419 168 168 168 305 305 305 F F F
181642024-01-10T18:02:54.048Z 21 420-439 167 167 167 303 303 303 F F F
181652024-01-10T18:02:54.048Z 22 440-459 159 159 159 283 283 283 F F F
181662024-01-10T18:02:54.048Z 23 460-479 168 168 168 304 304 304 F F F
181672024-01-10T18:02:54.050Z 24 480-499 162 162 162 292 292 292 F F F
181682024-01-10T18:02:54.050Z 25 500-519 160 160 160 284 284 284 F F F
181692024-01-10T18:02:54.050Z 26 520-539 168 168 168 306 306 306 F F F
181702024-01-10T18:02:54.050Z 27 540-559 168 168 168 306 306 306 F F F
181712024-01-10T18:02:54.050Z 28 560-579 169 169 169 307 307 307 F F F
181722024-01-10T18:02:54.050Z 29 580-599 164 164 164 295 295 295 F F F
181732024-01-10T18:02:54.050ZMax gen: 169, Max flush: 307
181742024-01-10T18:02:54.050Z
181752024-01-10T18:02:54.050ZBegin loop 85
181762024-01-10T18:02:54.050ZDownstairs to restart: 2
181772024-01-10T18:02:54.050Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.31121985Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838}
181782024-01-10T18:02:54.050Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.311659875Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838}
181792024-01-10T18:02:54.050Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.311709548Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838}
181802024-01-10T18:02:54.050Z{"msg":"Crucible fda2cdf8-e143-4e71-8f8b-b3eb896ce652 has session id: 6d80ed1f-80e9-40f7-b143-57724558bbec","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.311732416Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181812024-01-10T18:02:54.050Z{"msg":"Upstairs opts: Upstairs UUID: fda2cdf8-e143-4e71-8f8b-b3eb896ce652, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.311831479Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181822024-01-10T18:02:54.050Z{"msg":"Crucible stats registered with UUID: fda2cdf8-e143-4e71-8f8b-b3eb896ce652","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.311876104Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181832024-01-10T18:02:54.051ZCrucible runtime is spawned
181842024-01-10T18:02:54.051Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.312222117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838}
181852024-01-10T18:02:54.051Z{"msg":"fda2cdf8-e143-4e71-8f8b-b3eb896ce652 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.312283172Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181862024-01-10T18:02:54.051Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.312308457Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181872024-01-10T18:02:54.051Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.312423073Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181882024-01-10T18:02:54.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.312547823Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181892024-01-10T18:02:54.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.312652857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181902024-01-10T18:02:54.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.31269941Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181912024-01-10T18:02:54.053Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.31274237Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181922024-01-10T18:02:54.053Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.312883852Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181932024-01-10T18:02:54.053Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.312962591Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181942024-01-10T18:02:54.053Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.313041493Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181952024-01-10T18:02:54.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.313078047Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181962024-01-10T18:02:54.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.313144939Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181972024-01-10T18:02:54.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.313254592Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181982024-01-10T18:02:54.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.314285704Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
181992024-01-10T18:02:54.053Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.314368515Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182002024-01-10T18:02:54.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.314518191Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182012024-01-10T18:02:54.053Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.314600025Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182022024-01-10T18:02:54.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.315130018Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182032024-01-10T18:02:54.053Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.315206655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182042024-01-10T18:02:54.054Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.315233979Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182052024-01-10T18:02:54.055Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.315304954Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182062024-01-10T18:02:54.055Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.316220645Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182072024-01-10T18:02:54.055Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.316319027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182082024-01-10T18:02:54.055Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.316360624Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182092024-01-10T18:02:54.055Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.31638266Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182102024-01-10T18:02:54.055Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.316441022Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182112024-01-10T18:02:54.055Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.316465261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182122024-01-10T18:02:54.055Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.31648693Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182132024-01-10T18:02:54.055Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.31711922Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182142024-01-10T18:02:54.056Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317244105Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182152024-01-10T18:02:54.056Z{"msg":"[0]R flush_numbers[0..12]: [304, 305, 302, 306, 292, 291, 285, 304, 306, 306, 303, 305]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.31728759Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182162024-01-10T18:02:54.056Z{"msg":"[0]R generation[0..12]: [168, 168, 166, 168, 162, 162, 160, 168, 168, 168, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317338595Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182172024-01-10T18:02:54.056Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317364194Z"Wait for a query_work_queue command to finish before sending IO
182182024-01-10T18:02:54.057Z,"hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182192024-01-10T18:02:54.057Z{"msg":"[1]R flush_numbers[0..12]: [304, 305, 302, 306, 292, 291, 285, 304, 306, 306, 303, 305]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317470833Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182202024-01-10T18:02:54.057Z{"msg":"[1]R generation[0..12]: [168, 168, 166, 168, 162, 162, 160, 168, 168, 168, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317547119Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182212024-01-10T18:02:54.057Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317595385Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182222024-01-10T18:02:54.057Z{"msg":"[2]R flush_numbers[0..12]: [304, 305, 302, 306, 292, 291, 285, 304, 306, 306, 303, 305]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317620027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182232024-01-10T18:02:54.057Z{"msg":"[2]R generation[0..12]: [168, 168, 166, 168, 162, 162, 160, 168, 168, 168, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317677358Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182242024-01-10T18:02:54.057Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317717248Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182252024-01-10T18:02:54.057Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317776222Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182262024-01-10T18:02:54.057Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317801657Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182272024-01-10T18:02:54.058Z{"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317857567Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182282024-01-10T18:02:54.058Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317883325Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182292024-01-10T18:02:54.058Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317941714Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182302024-01-10T18:02:54.058Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.317968681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182312024-01-10T18:02:54.058Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.318023301Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182322024-01-10T18:02:54.058Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.318052452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182332024-01-10T18:02:54.058Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.318106406Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182342024-01-10T18:02:54.058Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.318130025Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182352024-01-10T18:02:54.058Z{"msg":"fda2cdf8-e143-4e71-8f8b-b3eb896ce652 is now active with session: 6d80ed1f-80e9-40f7-b143-57724558bbec","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.318185626Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182362024-01-10T18:02:54.058Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.318213261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838,"session_id":"6d80ed1f-80e9-40f7-b143-57724558bbec"}
182372024-01-10T18:02:54.058Z{"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
182382024-01-10T18:02:54.058Zmsg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:40.31826967Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1838}
182392024-01-10T18:02:54.058ZLoading write count information from file "/tmp/test_repair_verify.data"
182402024-01-10T18:02:54.058ZRun Repair workload
182412024-01-10T18:02:54.059Z01/30 Flush
182422024-01-10T18:02:54.154Z02/30 Write block 497 len 20480 data: 17 17 16 14 13
182432024-01-10T18:02:54.154Z03/30 Read block 297 len 16384
182442024-01-10T18:02:54.154Z04/30 Read block 106 len 32768
182452024-01-10T18:02:54.154Z05/30 Read block 44 len 24576
182462024-01-10T18:02:54.154Z06/30 Read block 285 len 4096
182472024-01-10T18:02:54.154Z07/30 Write block 280 len 24576 data: 13 13 11 12 14 16
182482024-01-10T18:02:54.154Z08/30 Read block 447 len 32768
182492024-01-10T18:02:54.154Z09/30 Read block 503 len 28672
182502024-01-10T18:02:54.154Z10/30 Flush
182512024-01-10T18:02:54.154Z11/30 Write block 466 len 36864 data: 13 14 15 12 12 8 7 8 9
182522024-01-10T18:02:54.154Z12/30 Read block 90 len 24576
182532024-01-10T18:02:54.154Z13/30 Read block 346 len 36864
182542024-01-10T18:02:54.154Z14/30 Read block 453 len 8192
182552024-01-10T18:02:54.155Z15/30 Read block 240 len 28672
182562024-01-10T18:02:54.155Z16/30 Flush
182572024-01-10T18:02:54.155Z17/30 Write block 20 len 32768 data: 13 14 14 16 12 13 13 12
182582024-01-10T18:02:54.155Z18/30 Read block 330 len 4096
182592024-01-10T18:02:54.155Z19/30 Read block 524 len 28672
182602024-01-10T18:02:54.155Z20/30 Flush
182612024-01-10T18:02:54.155Z21/30 Write block 110 len 4096 data: 11
182622024-01-10T18:02:54.155Z22/30 Write block 17 len 4096 data: 13
182632024-01-10T18:02:54.155Z23/30 Read block 135 len 24576
182642024-01-10T18:02:54.155Z24/30 Flush
182652024-01-10T18:02:54.155Z25/30 Write block 377 len 36864 data: 8 7 8 8 9 9 6 7 8
182662024-01-10T18:02:54.155Z26/30 Read block 310 len 12288
182672024-01-10T18:02:54.155Z27/30 Write block 205 len 16384 data: 11 11 11 12
182682024-01-10T18:02:54.155Z28/30 Read block 423 len 40960
182692024-01-10T18:02:54.156Z29/30 Write block 384 len 24576 data: 8 9 8 7 6 8
182702024-01-10T18:02:54.156Z30/30 Read block 528 len 12288
182712024-01-10T18:02:54.157Z----------------------------------------------------------------
182722024-01-10T18:02:54.157Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:29
182732024-01-10T18:02:54.157ZStates: Active Active Active
182742024-01-10T18:02:54.157ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182752024-01-10T18:02:54.157Z 2 Acked 1001 Write 5 Done Done Done false
182762024-01-10T18:02:54.157Z 3 Acked 1002 Read 4 Done Done Done false
182772024-01-10T18:02:54.157Z 4 Acked 1003 Read 8 Done Done Sent false
182782024-01-10T18:02:54.157Z 5 Acked 1004 Read 6 Done Done Sent false
182792024-01-10T18:02:54.157Z 6 Acked 1005 Read 1 Done Done Sent false
182802024-01-10T18:02:54.157Z 7 Acked 1006 Write 6 Done Done Sent false
182812024-01-10T18:02:54.158Z 8 Acked 1007 Read 8 Done Done Sent false
182822024-01-10T18:02:54.158Z 9 Acked 1008 Read 7 Done Done Sent false
182832024-01-10T18:02:54.158Z 10 Acked 1009 Flush 0 Done Done Sent false
182842024-01-10T18:02:54.158Z 11 Acked 1010 Write 9 Done Done Sent false
182852024-01-10T18:02:54.158Z 12 Acked 1011 Read 6 Done Done Sent false
182862024-01-10T18:02:54.158Z 13 Acked 1012 Read 9 Done Done Sent false
182872024-01-10T18:02:54.158Z 14 Acked 1013 Read 2 Done Done Sent false
182882024-01-10T18:02:54.158Z 15 Acked 1014 Read 7 Done Done Sent false
182892024-01-10T18:02:54.158Z 16 Acked 1015 Flush 0 Done Done Sent false
182902024-01-10T18:02:54.158Z 17 Acked 1016 Write 8 Done Done Sent false
182912024-01-10T18:02:54.158Z 18 Acked 1017 Read 1 Done Done Sent false
182922024-01-10T18:02:54.158Z 19 Acked 1018 Read 7 Done Done Sent false
182932024-01-10T18:02:54.158Z 20 Acked 1019 Flush 0 Done Done Sent false
182942024-01-10T18:02:54.159Z 21 Acked 1020 Write 1 Done Done Sent false
182952024-01-10T18:02:54.159Z 22 Acked 1021 Write 1 Done Done Sent false
182962024-01-10T18:02:54.159Z 23 Acked 1022 Read 6 Done Done Sent false
182972024-01-10T18:02:54.159Z 24 Acked 1023 Flush 0 Done Done Sent false
182982024-01-10T18:02:54.159Z 25 Acked 1024 Write 9 Done Done Sent false
182992024-01-10T18:02:54.159Z 26 Acked 1025 Read 3 Done Done Sent false
183002024-01-10T18:02:54.159Z 27 Acked 1026 Write 4 Done Done Sent false
183012024-01-10T18:02:54.159Z 28 Acked 1027 Read 10 Done Done Sent false
183022024-01-10T18:02:54.159Z 29 Acked 1028 Write 6 Done Done Sent false
183032024-01-10T18:02:54.159Z 30 Acked 1029 Read 3 Done Done Sent false
183042024-01-10T18:02:54.159Z STATES DS:0 DS:1 DS:2 TOTAL
183052024-01-10T18:02:54.159Z New 0 0 0 0
183062024-01-10T18:02:54.159Z Sent 0 0 27 27
183072024-01-10T18:02:54.159Z Done 29 29 2 60
183082024-01-10T18:02:54.160Z Skipped 0 0 0 0
183092024-01-10T18:02:54.160Z Error 0 0 0 0
183102024-01-10T18:02:54.160ZLast Flush: 1023 1023 1000
183112024-01-10T18:02:54.160ZDownstairs last five completed: 1000
183122024-01-10T18:02:54.160ZUpstairs last five completed: 30 29 28 27 26
183132024-01-10T18:02:54.160ZWrote out file "/tmp/test_repair_verify.data"
183142024-01-10T18:02:54.160Z
183152024-01-10T18:02:54.161ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
183162024-01-10T18:02:54.161ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183172024-01-10T18:02:54.161Z 0 000-019 170 170 168 312 312 304 F F F <---
183182024-01-10T18:02:54.161Z 1 020-039 170 170 168 311 311 305 F F F <---
183192024-01-10T18:02:54.162Z 2 040-059 166 166 166 302 302 302 F F F
183202024-01-10T18:02:54.162Z 3 060-079 168 168 168 306 306 306 F F F
183212024-01-10T18:02:54.162Z 4 080-099 162 162 162 292 292 292 F F F
183222024-01-10T18:02:54.162Z 5 100-119 170 170 162 312 312 291 F F F <---
183232024-01-10T18:02:54.162Z 6 120-139 160 160 160 285 285 285 F F F
183242024-01-10T18:02:54.162Z 7 140-159 168 168 168 304 304 304 F F F
183252024-01-10T18:02:54.162Z 8 160-179 168 168 168 306 306 306 F F F
183262024-01-10T18:02:54.162Z 9 180-199 168 168 168 306 306 306 F F F
183272024-01-10T18:02:54.162Z 10 200-219 167 167 167 303 303 303 T T F <---
183282024-01-10T18:02:54.162Z 11 220-239 168 168 168 305 305 305 F F F
183292024-01-10T18:02:54.162Z 12 240-259 166 166 166 300 300 300 F F F
183302024-01-10T18:02:54.162Z 13 260-279 165 165 165 297 297 297 F F F
183312024-01-10T18:02:54.162Z 14 280-299 170 170 166 309 309 300 F F F <---
183322024-01-10T18:02:54.163Z 15 300-319 169 169 169 307 307 307 F F F
183332024-01-10T18:02:54.163Z 16 320-339 155 155 155 274 274 274 F F F
183342024-01-10T18:02:54.163Z 17 340-359 167 167 167 303 303 303 F F F
183352024-01-10T18:02:54.163Z 18 360-379 162 162 162 292 292 292 T T F <---
183362024-01-10T18:02:54.163Z 19 380-399 166 166 166 300 300 300 T T F <---
183372024-01-10T18:02:54.163Z 20 400-419 168 168 168 305 305 305 F F F
183382024-01-10T18:02:54.163Z 21 420-439 167 167 167 303 303 303 F F F
183392024-01-10T18:02:54.163Z 22 440-459 159 159 159 283 283 283 F F F
183402024-01-10T18:02:54.163Z 23 460-479 170 170 168 310 310 304 F F F <---
183412024-01-10T18:02:54.163Z 24 480-499 170 170 162 309 309 292 F F T <---
183422024-01-10T18:02:54.247Z 25 500-519 170 170 160 309 309 284 F F T <---
183432024-01-10T18:02:54.247Z 26 520-539 168 168 168 306 306 306 F F F
183442024-01-10T18:02:54.248Z 27 540-559 168 168 168 306 306 306 F F F
183452024-01-10T18:02:54.248Z 28 560-579 169 169 169 307 307 307 F F F
183462024-01-10T18:02:54.248Z 29 580-599 164 164 164 295 295 295 F F F
183472024-01-10T18:02:54.248ZMax gen: 170, Max flush: 312
183482024-01-10T18:02:54.248ZError: Difference in extent metadata found!
183492024-01-10T18:02:54.249ZA Difference in extent metadata is expected here
183502024-01-10T18:02:54.249ZOn loop 85
183512024-01-10T18:02:54.249Z
183522024-01-10T18:02:54.249ZVerifying data now
183532024-01-10T18:02:54.249ZNo change after verify
183542024-01-10T18:02:54.249ZLoop: 85 Downstairs dump after verify (and repair):
183552024-01-10T18:02:54.250ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183562024-01-10T18:02:54.250Z 0 000-019 170 170 170 312 312 312 F F F
183572024-01-10T18:02:54.250Z 1 020-039 170 170 170 311 311 311 F F F
183582024-01-10T18:02:54.250Z 2 040-059 166 166 166 302 302 302 F F F
183592024-01-10T18:02:54.250Z 3 060-079 168 168 168 306 306 306 F F F
183602024-01-10T18:02:54.250Z 4 080-099 162 162 162 292 292 292 F F F
183612024-01-10T18:02:54.251Z 5 100-119 170 170 170 312 312 312 F F F
183622024-01-10T18:02:54.251Z 6 120-139 160 160 160 285 285 285 F F F
183632024-01-10T18:02:54.251Z 7 140-159 168 168 168 304 304 304 F F F
183642024-01-10T18:02:54.251Z 8 160-179 168 168 168 306 306 306 F F F
183652024-01-10T18:02:54.251Z 9 180-199 168 168 168 306 306 306 F F F
183662024-01-10T18:02:54.251Z 10 200-219 171 171 171 313 313 313 F F F
183672024-01-10T18:02:54.251Z 11 220-239 168 168 168 305 305 305 F F F
183682024-01-10T18:02:54.251Z 12 240-259 166 166 166 300 300 300 F F F
183692024-01-10T18:02:54.251Z 13 260-279 165 165 165 297 297 297 F F F
183702024-01-10T18:02:54.251Z 14 280-299 170 170 170 309 309 309 F F F
183712024-01-10T18:02:54.252Z 15 300-319 169 169 169 307 307 307 F F F
183722024-01-10T18:02:54.252Z 16 320-339 155 155 155 274 274 274 F F F
183732024-01-10T18:02:54.252Z 17 340-359 167 167 167 303 303 303 F F F
183742024-01-10T18:02:54.252Z 18 360-379 171 171 171 313 313 313 F F F
183752024-01-10T18:02:54.252Z 19 380-399 171 171 171 313 313 313 F F F
183762024-01-10T18:02:54.252Z 20 400-419 168 168 168 305 305 305 F F F
183772024-01-10T18:02:54.254Z 21 420-439 167 167 167 303 303 303 F F F
183782024-01-10T18:02:54.254Z 22 440-459 159 159 159 283 283 283 F F F
183792024-01-10T18:02:54.254Z 23 460-479 170 170 170 310 310 310 F F F
183802024-01-10T18:02:54.255Z 24 480-499 170 170 170 309 309 309 F F F
183812024-01-10T18:02:54.255Z 25 500-519 170 170 170 309 309 309 F F F
183822024-01-10T18:02:54.256Z 26 520-539 168 168 168 306 306 306 F F F
183832024-01-10T18:02:54.256Z 27 540-559 168 168 168 306 306 306 F F F
183842024-01-10T18:02:54.256Z 28 560-579 169 169 169 307 307 307 F F F
183852024-01-10T18:02:54.256Z 29 580-599 164 164 164 295 295 295 F F F
183862024-01-10T18:02:54.257ZMax gen: 171, Max flush: 313
183872024-01-10T18:02:54.257Z
183882024-01-10T18:02:54.257ZBegin loop 86
183892024-01-10T18:02:54.257ZDownstairs to restart: 1
183902024-01-10T18:02:54.257Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.528096508Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846}
183912024-01-10T18:02:54.258Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.528568472Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846}
183922024-01-10T18:02:54.258Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.528596244Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846}
183932024-01-10T18:02:54.258Z{"msg":"Crucible db17aa73-593c-4c33-a450-3ae8ced7d4d4 has session id: dedaf5a9-111e-42aa-860e-a7036baf4b83","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.528616759Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
183942024-01-10T18:02:54.258Z{"msg":"Upstairs opts: Upstairs UUID: db17aa73-593c-4c33-a450-3ae8ced7d4d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.528651437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
183952024-01-10T18:02:54.261Z{"msg":"Crucible stats registered with UUID: db17aa73-593c-4c33-a450-3ae8ced7d4d4","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.528679166Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
183962024-01-10T18:02:54.261ZCrucible runtime is spawned
183972024-01-10T18:02:54.261Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.528901457Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846}
183982024-01-10T18:02:54.261Z{"msg":"db17aa73-593c-4c33-a450-3ae8ced7d4d4 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529099426Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
183992024-01-10T18:02:54.261Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529127447Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184002024-01-10T18:02:54.261Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529160747Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184012024-01-10T18:02:54.261Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529203215Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184022024-01-10T18:02:54.261Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529245611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184032024-01-10T18:02:54.261Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529285684Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184042024-01-10T18:02:54.261Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529324476Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184052024-01-10T18:02:54.262Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529364157Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184062024-01-10T18:02:54.262Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529407463Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184072024-01-10T18:02:54.262Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529450277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184082024-01-10T18:02:54.262Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529612194Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184092024-01-10T18:02:54.262Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529665962Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184102024-01-10T18:02:54.262Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.529711761Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184112024-01-10T18:02:54.262Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.53093709Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184122024-01-10T18:02:54.262Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.530976629Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184132024-01-10T18:02:54.262Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.531000006Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184142024-01-10T18:02:54.262Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.531066003Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184152024-01-10T18:02:54.263Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.531089973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184162024-01-10T18:02:54.263Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.531112016Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184172024-01-10T18:02:54.263Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.531818273Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184182024-01-10T18:02:54.263Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.532061711Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184192024-01-10T18:02:54.263Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.532173059Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184202024-01-10T18:02:54.263Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.532734526Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184212024-01-10T18:02:54.263Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.532781388Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184222024-01-10T18:02:54.263Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.532806542Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184232024-01-10T18:02:54.263Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.53306166Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184242024-01-10T18:02:54.263Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533106635Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184252024-01-10T18:02:54.265Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533131495Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184262024-01-10T18:02:54.265Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.53315214Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184272024-01-10T18:02:54.265Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533174425Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184282024-01-10T18:02:54.265Z{"msg":"[0]R flush_numbers[0..12]: [312, 311, 302, 306, 292, 312, 285, 304, 306, 306, 313, 305]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533198577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184292024-01-10T18:02:54.265Z{"msg":"[0]R generation[0..12]: [170, 170, 166, 168, 162, 170, 160, 168, 168, 168, 171, 168]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533222371Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184302024-01-10T18:02:54.265Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533246525Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184312024-01-10T18:02:54.265Z{"msg":"[1]R flush_numbers[0..12]: [312, 311, 302, 306, 292, 312, 285, 304, 306, 306, 313, 305]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.53327016Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184322024-01-10T18:02:54.265Z{"msg":"[1]R generation[0..12]: [170, 170, 166, 168, 162, 170, 160, 168, 168, 168, 171, 168]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533294502Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184332024-01-10T18:02:54.265Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533328005Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184342024-01-10T18:02:54.265Z{"msg":"[2]R flush_numbers[0..12]: [312, 311, 302, 306, 292, 312, 285, 304, 306, 306, 313, 305]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533352654Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184352024-01-10T18:02:54.265Z{"msg":"[2]R generation[0..12]: [170, 170, 166, 168, 162, 170, 160, 168, 168, 168, 171, 168]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533375348Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184362024-01-10T18:02:54.265ZWait for a query_work_queue command to finish before sending IO
184372024-01-10T18:02:54.266Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533401154Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184382024-01-10T18:02:54.266Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533422656Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184392024-01-10T18:02:54.266Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533444591Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184402024-01-10T18:02:54.266Z{"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533465577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184412024-01-10T18:02:54.266Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.53348658Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184422024-01-10T18:02:54.298Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533507952Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184432024-01-10T18:02:54.299Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533527587Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184442024-01-10T18:02:54.299Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:41.533549071Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184452024-01-10T18:02:54.299ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184462024-01-10T18:02:54.299Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:53.933688915Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184472024-01-10T18:02:54.299Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:53.933759655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184482024-01-10T18:02:54.299Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:53.933782162Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184492024-01-10T18:02:54.299Z{"msg":"db17aa73-593c-4c33-a450-3ae8ced7d4d4 is now active with session: dedaf5a9-111e-42aa-860e-a7036baf4b83","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:53.933804333Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184502024-01-10T18:02:54.299Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:53.933845029Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846,"session_id":"dedaf5a9-111e-42aa-860e-a7036baf4b83"}
184512024-01-10T18:02:54.299Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:53.933868684Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1846}
184522024-01-10T18:02:54.299ZLoading write count information from file "/tmp/test_repair_verify.data"
184532024-01-10T18:02:54.299ZRun Repair workload
184542024-01-10T18:02:54.299Z01/30 Write block 545 len 12288 data: 14 13 16
184552024-01-10T18:02:54.299Z02/30 Read block 397 len 28672
184562024-01-10T18:02:54.299Z03/30 Write block 55 len 36864 data: 14 13 12 17 14 14 12 12 13
184572024-01-10T18:02:54.299Z04/30 Read block 480 len 28672
184582024-01-10T18:02:54.299Z05/30 Write block 536 len 24576 data: 17 16 15 16 19 20
184592024-01-10T18:02:54.299Z06/30 Read block 426 len 40960
184602024-01-10T18:02:54.299Z07/30 Read block 377 len 36864
184612024-01-10T18:02:54.299Z08/30 Flush
184622024-01-10T18:02:54.299Z09/30 Write block 561 len 4096 data: 12
184632024-01-10T18:02:54.300Z10/30 Read block 315 len 8192
184642024-01-10T18:02:54.300Z11/30 Write block 115 len 16384 data: 12 14 15 13
184652024-01-10T18:02:54.300Z12/30 Write block 278 len 40960 data: 14 13 14 14 12 13 15 17 14 15
184662024-01-10T18:02:54.300Z13/30 Read block 112 len 40960
184672024-01-10T18:02:54.301Z14/30 Read block 570 len 4096
184682024-01-10T18:02:54.301Z15/30 Flush
184692024-01-10T18:02:54.301Z16/30 Write block 155 len 4096 data: 13
184702024-01-10T18:02:54.301Z17/30 Write block 265 len 16384 data: 11 13 15 13
184712024-01-10T18:02:54.301Z18/30 Write block 387 len 16384 data: 8 7 9 10
184722024-01-10T18:02:54.301Z19/30 Read block 71 len 28672
184732024-01-10T18:02:54.301Z20/30 Read block 553 len 24576
184742024-01-10T18:02:54.301Z21/30 Write block 13 len 40960 data: 8 11 12 13 14 15 14 14 15 15
184752024-01-10T18:02:54.302Z22/30 Flush
184762024-01-10T18:02:54.302Z23/30 Write block 94 len 28672 data: 13 15 16 16 12 14 13
184772024-01-10T18:02:54.302Z24/30 Flush
184782024-01-10T18:02:54.302Z25/30 Write block 197 len 28672 data: 18 19 19 17 16 15 12
184792024-01-10T18:02:54.302Z26/30 Read block 479 len 16384
184802024-01-10T18:02:54.302Z27/30 Write block 462 len 24576 data: 9 12 12 10 14 15
184812024-01-10T18:02:54.302Z28/30 Read block 539 len 16384
184822024-01-10T18:02:54.302Z29/30 Read block 515 len 24576
184832024-01-10T18:02:54.302Z30/30 Write block 421 len 40960 data: 20 19 17 17 16 15 14 14 13 11
184842024-01-10T18:02:54.302Z----------------------------------------------------------------
184852024-01-10T18:02:54.302Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
184862024-01-10T18:02:54.302ZStates: Active Active Active
184872024-01-10T18:02:54.302ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
184882024-01-10T18:02:54.302Z 1 Acked 1000 Write 3 Done Done Done false
184892024-01-10T18:02:54.303Z 2 Acked 1001 Read 7 Done Done Done false
184902024-01-10T18:02:54.303Z 3 Acked 1002 Write 9 Done Done Done false
184912024-01-10T18:02:54.303Z 4 Acked 1003 Read 7 Done Done Done false
184922024-01-10T18:02:54.303Z 5 Acked 1004 Write 6 Done Done Done false
184932024-01-10T18:02:54.303Z 6 Acked 1005 Read 10 Done Done Done false
184942024-01-10T18:02:54.303Z 7 Acked 1006 Read 9 Done Done Done false
184952024-01-10T18:02:54.303Z 8 Acked 1007 Flush 0 Done Sent Done false
184962024-01-10T18:02:54.303Z 9 Acked 1008 Write 1 Done Sent Done false
184972024-01-10T18:02:54.303Z 10 Acked 1009 Read 2 Done Sent Done false
184982024-01-10T18:02:54.303Z 11 Acked 1010 Write 4 Done Sent Done false
184992024-01-10T18:02:54.306Z 12 Acked 1011 Write 10 Done Sent Done false
185002024-01-10T18:02:54.306Z 13 Acked 1012 Read 10 Done Sent Done false
185012024-01-10T18:02:54.306Z 14 Acked 1013 Read 1 Done Sent Done false
185022024-01-10T18:02:54.306Z 15 Acked 1014 Flush 0 Done Sent Done false
185032024-01-10T18:02:54.306Z 16 Acked 1015 Write 1 Done Sent Done false
185042024-01-10T18:02:54.306Z 17 Acked 1016 Write 4 Done Sent Done false
185052024-01-10T18:02:54.306Z 18 Acked 1017 Write 4 Done Sent Done false
185062024-01-10T18:02:54.306Z 19 Acked 1018 Read 7 Done Sent Done false
185072024-01-10T18:02:54.306Z 20 Acked 1019 Read 6 Done Sent Done false
185082024-01-10T18:02:54.306Z 21 Acked 1020 Write 10 Done Sent Done false
185092024-01-10T18:02:54.306Z 22 Acked 1021 Flush 0 Done Sent Done false
185102024-01-10T18:02:54.306Z 23 Acked 1022 Write 7 Done Sent Done false
185112024-01-10T18:02:54.306Z 24 Acked 1023 Flush 0 Done Sent Done false
185122024-01-10T18:02:54.306Z 25 Acked 1024 Write 7 Done Sent Done false
185132024-01-10T18:02:54.306Z 26 Acked 1025 Read 4 Done Sent Done false
185142024-01-10T18:02:54.306Z 27 Acked 1026 Write 6 Done Sent Done false
185152024-01-10T18:02:54.306Z 28 Acked 1027 Read 4 Done Sent Done false
185162024-01-10T18:02:54.306Z 29 Acked 1028 Read 6 Done Sent Done false
185172024-01-10T18:02:54.307Z 30 Acked 1029 Write 10 Sent Sent Sent false
185182024-01-10T18:02:54.307Z STATES DS:0 DS:1 DS:2 TOTAL
185192024-01-10T18:02:54.307Z New 0 0 0 0
185202024-01-10T18:02:54.307Z Sent 1 23 1 25
185212024-01-10T18:02:54.307Z Done 29 7 29 65
185222024-01-10T18:02:54.307Z Skipped 0 0 0 0
185232024-01-10T18:02:54.307Z Error 0 0 0 0
185242024-01-10T18:02:54.307ZLast Flush: 1023 0 1023
185252024-01-10T18:02:54.307ZDownstairs last five completed:
185262024-01-10T18:02:54.307ZUpstairs last five completed: 30 29 28 27 26
185272024-01-10T18:02:54.307ZWrote out file "/tmp/test_repair_verify.data"
185282024-01-10T18:02:54.307Z
185292024-01-10T18:02:54.307ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185302024-01-10T18:02:54.307ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185312024-01-10T18:02:54.307Z 0 000-019 172 170 172 316 312 316 F F F <---
185322024-01-10T18:02:54.307Z 1 020-039 172 170 172 316 311 316 F F F <---
185332024-01-10T18:02:54.307Z 2 040-059 172 166 172 314 302 314 F T F <---
185342024-01-10T18:02:54.307Z 3 060-079 172 168 172 314 306 314 F T F <---
185352024-01-10T18:02:54.308Z 4 080-099 172 162 172 317 292 317 F F F <---
185362024-01-10T18:02:54.308Z 5 100-119 172 170 172 317 312 317 F F F <---
185372024-01-10T18:02:54.308Z 6 120-139 160 160 160 285 285 285 F F F
185382024-01-10T18:02:54.308Z 7 140-159 172 168 172 316 304 316 F F F <---
185392024-01-10T18:02:54.308Z 8 160-179 168 168 168 306 306 306 F F F
185402024-01-10T18:02:54.308Z 9 180-199 168 168 168 306 306 306 T F T <---
185412024-01-10T18:02:54.308Z 10 200-219 171 171 171 313 313 313 T F T <---
185422024-01-10T18:02:54.330Z 11 220-239 168 168 168 305 305 305 F F F
185432024-01-10T18:02:54.330Z 12 240-259 166 166 166 300 300 300 F F F
185442024-01-10T18:02:54.330Z 13 260-279 172 165 172 316 297 316 F F F <---
185452024-01-10T18:02:54.331Z 14 280-299 172 170 172 315 309 315 F F F <---
185462024-01-10T18:02:54.332Z 15 300-319 169 169 169 307 307 307 F F F
185472024-01-10T18:02:54.332Z 16 320-339 155 155 155 274 274 274 F F F
185482024-01-10T18:02:54.332Z 17 340-359 167 167 167 303 303 303 F F F
185492024-01-10T18:02:54.332Z 18 360-379 171 171 171 313 313 313 F F F
185502024-01-10T18:02:54.333Z 19 380-399 172 171 172 316 313 316 F F F <---
185512024-01-10T18:02:54.333Z 20 400-419 168 168 168 305 305 305 F F F
185522024-01-10T18:02:54.333Z 21 420-439 167 167 167 303 303 303 T F T <---
185532024-01-10T18:02:54.333Z 22 440-459 159 159 159 283 283 283 F F F
185542024-01-10T18:02:54.333Z 23 460-479 170 170 170 310 310 310 T F T <---
185552024-01-10T18:02:54.333Z 24 480-499 170 170 170 309 309 309 F F F
185562024-01-10T18:02:54.333Z 25 500-519 170 170 170 309 309 309 F F F
185572024-01-10T18:02:54.333Z 26 520-539 172 168 172 314 306 314 F T F <---
185582024-01-10T18:02:54.333Z 27 540-559 172 168 172 314 306 314 F T F <---
185592024-01-10T18:02:54.333Z 28 560-579 172 169 172 315 307 315 F F F <---
185602024-01-10T18:02:54.334Z 29 580-599 164 164 164 295 295 295 F F F
185612024-01-10T18:02:54.334ZMax gen: 172, Max flush: 317
185622024-01-10T18:02:54.334ZError: Difference in extent metadata found!
185632024-01-10T18:02:54.334ZA Difference in extent metadata is expected here
185642024-01-10T18:02:54.334ZOn loop 86
185652024-01-10T18:02:54.334Z
185662024-01-10T18:02:54.334ZVerifying data now
185672024-01-10T18:02:56.626ZNo change after verify
185682024-01-10T18:02:56.626ZLoop: 86 Downstairs dump after verify (and repair):
185692024-01-10T18:02:56.653ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185702024-01-10T18:02:56.654Z 0 000-019 172 172 172 316 316 316 F F F
185712024-01-10T18:02:56.654Z 1 020-039 172 172 172 316 316 316 F F F
185722024-01-10T18:02:56.654Z 2 040-059 172 172 172 314 314 314 F F F
185732024-01-10T18:02:56.654Z 3 060-079 172 172 172 314 314 314 F F F
185742024-01-10T18:02:56.654Z 4 080-099 172 172 172 317 317 317 F F F
185752024-01-10T18:02:56.654Z 5 100-119 172 172 172 317 317 317 F F F
185762024-01-10T18:02:56.654Z 6 120-139 160 160 160 285 285 285 F F F
185772024-01-10T18:02:56.673Z 7 140-159 172 172 172 316 316 316 F F F
185782024-01-10T18:02:56.673Z 8 160-179 168 168 168 306 306 306 F F F
185792024-01-10T18:02:56.673Z 9 180-199 173 173 173 318 318 318 F F F
185802024-01-10T18:02:56.673Z 10 200-219 173 173 173 318 318 318 F F F
185812024-01-10T18:02:56.673Z 11 220-239 168 168 168 305 305 305 F F F
185822024-01-10T18:02:56.673Z 12 240-259 166 166 166 300 300 300 F F F
185832024-01-10T18:02:56.674Z 13 260-279 172 172 172 316 316 316 F F F
185842024-01-10T18:02:56.674Z 14 280-299 172 172 172 315 315 315 F F F
185852024-01-10T18:02:56.674Z 15 300-319 169 169 169 307 307 307 F F F
185862024-01-10T18:02:56.674Z 16 320-339 155 155 155 274 274 274 F F F
185872024-01-10T18:02:56.675Z 17 340-359 167 167 167 303 303 303 F F F
185882024-01-10T18:02:56.675Z 18 360-379 171 171 171 313 313 313 F F F
185892024-01-10T18:02:56.675Z 19 380-399 172 172 172 316 316 316 F F F
185902024-01-10T18:02:56.675Z 20 400-419 168 168 168 305 305 305 F F F
185912024-01-10T18:02:56.675Z 21 420-439 173 173 173 318 318 318 F F F
185922024-01-10T18:02:56.675Z 22 440-459 159 159 159 283 283 283 F F F
185932024-01-10T18:02:56.676Z 23 460-479 173 173 173 318 318 318 F F F
185942024-01-10T18:02:56.676Z 24 480-499 170 170 170 309 309 309 F F F
185952024-01-10T18:02:56.676Z 25 500-519 170 170 170 309 309 309 F F F
185962024-01-10T18:02:56.676Z 26 520-539 172 172 172 314 314 314 F F F
185972024-01-10T18:02:56.678Z 27 540-559 172 172 172 314 314 314 F F F
185982024-01-10T18:02:56.679Z 28 560-579 172 172 172 315 315 315 F F F
185992024-01-10T18:02:56.679Z 29 580-599 164 164 164 295 295 295 F F F
186002024-01-10T18:02:56.680ZMax gen: 173, Max flush: 318
186012024-01-10T18:02:56.680Z
186022024-01-10T18:02:56.680ZBegin loop 87
186032024-01-10T18:02:56.680ZDownstairs to restart: 1
186042024-01-10T18:02:56.692Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.666187635Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854}
186052024-01-10T18:02:56.693Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.666745117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854}
186062024-01-10T18:02:56.693Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.666809788Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854}
186072024-01-10T18:02:56.693Z{"msg":"Crucible ccd689d4-d9f7-4d66-be1e-fcc0cc12fb12 has session id: 3014525c-cd80-4a7d-ad3b-b223600091c1","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.666839179Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186082024-01-10T18:02:56.693ZCrucible runtime is spawned
186092024-01-10T18:02:56.694Z{"msg":"Upstairs opts: Upstairs UUID: ccd689d4-d9f7-4d66-be1e-fcc0cc12fb12, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.666973532Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186102024-01-10T18:02:56.694Z{"msg":"Crucible stats registered with UUID: ccd689d4-d9f7-4d66-be1e-fcc0cc12fb12","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667068295Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186112024-01-10T18:02:56.694Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667109012Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854}
186122024-01-10T18:02:56.697Z{"msg":"ccd689d4-d9f7-4d66-be1e-fcc0cc12fb12 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667372811Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186132024-01-10T18:02:56.697Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667451618Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186142024-01-10T18:02:56.698Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667500127Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186152024-01-10T18:02:56.698Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667542808Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186162024-01-10T18:02:56.698Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667647939Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186172024-01-10T18:02:56.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667727546Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186182024-01-10T18:02:56.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667829478Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186192024-01-10T18:02:56.699Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667876679Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186202024-01-10T18:02:56.700Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.667997325Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186212024-01-10T18:02:56.700Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.668085456Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186222024-01-10T18:02:56.701Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.668191313Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186232024-01-10T18:02:56.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.668342839Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186242024-01-10T18:02:56.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.668456898Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186252024-01-10T18:02:56.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.669297377Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186262024-01-10T18:02:56.702Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.669380467Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186272024-01-10T18:02:56.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.669586593Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186282024-01-10T18:02:56.702Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.66965593Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186292024-01-10T18:02:56.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.669800227Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186302024-01-10T18:02:56.703Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.669858719Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186312024-01-10T18:02:56.703Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.67046688Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186322024-01-10T18:02:56.703Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.670592382Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186332024-01-10T18:02:56.703Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.671039882Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186342024-01-10T18:02:56.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.671383513Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186352024-01-10T18:02:56.703Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.671471035Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186362024-01-10T18:02:56.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.671546096Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186372024-01-10T18:02:56.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.671587238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186382024-01-10T18:02:56.704Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.671653958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186392024-01-10T18:02:56.704Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.671698152Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186402024-01-10T18:02:56.704Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.671878632Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186412024-01-10T18:02:56.704Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.67194138Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186422024-01-10T18:02:56.705Z{"msg":"[0]R flush_numbers[0..12]: [316, 316, 314, 314, 317, 317, 285, 316, 306, 318, 318, 305]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.671968658Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186432024-01-10T18:02:56.705Z{"msg":"[0]R generation[0..12]: [172, 172, 172, 172, 172, 172, 160, 172, 168, 173, 173, 168]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672028059Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186442024-01-10T18:02:56.705Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.67206202Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186452024-01-10T18:02:56.705Z{"msg":"[1]R flush_numbers[0..12]: [316, 316, 314, 314, 317, 317, 285, 316, 306, 318, 318, 305]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672116898Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186462024-01-10T18:02:56.705Z{"msg":"[1]R generation[0..12]: [172, 172, 172, 172, 172, 172, 160, 172, 168, 173, 173, 168]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
186472024-01-10T18:02:56.706Z,"time":"2024-01-10T18:02:56.672148574Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186482024-01-10T18:02:56.706Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672265731Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186492024-01-10T18:02:56.706Z{"msg":"[2]R flush_numbers[0..12]: [316, 316, 314, 314, 317, 317, 285, 316, 306, 318, 318, 305]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672332475Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186502024-01-10T18:02:56.706Z{"msg":"[2]R generation[0..12]: [172, 172, 172, 172, 172, 172, 160, 172, 168, 173, 173, 168]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672390173Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186512024-01-10T18:02:56.706Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672420503Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186522024-01-10T18:02:56.706Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672483779Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186532024-01-10T18:02:56.706Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.67250979Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186542024-01-10T18:02:56.706Z{"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672537624Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186552024-01-10T18:02:56.706Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672608575Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186562024-01-10T18:02:56.706Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672650878Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186572024-01-10T18:02:56.707Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672703238Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186582024-01-10T18:02:56.707Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672745636Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186592024-01-10T18:02:56.707Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.67283602Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186602024-01-10T18:02:56.707Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672870239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186612024-01-10T18:02:56.707Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
186622024-01-10T18:02:56.707Z,"time":"2024-01-10T18:02:56.672925391Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186632024-01-10T18:02:56.708Z{"msg":"ccd689d4-d9f7-4d66-be1e-fcc0cc12fb12 is now active with session: 3014525c-cd80-4a7d-ad3b-b223600091c1","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.672967427Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186642024-01-10T18:02:56.708Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.673034363Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854,"session_id":"3014525c-cd80-4a7d-ad3b-b223600091c1"}
186652024-01-10T18:02:56.708Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:02:56.673072784Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1854}
186662024-01-10T18:02:56.708ZLoading write count information from file "/tmp/test_repair_verify.data"
186672024-01-10T18:02:56.708ZRun Repair workload
186682024-01-10T18:02:56.708Z01/30 Write block 230 len 32768 data: 9 6 5 6 8 9 9 11
186692024-01-10T18:02:56.708Z02/30 Write block 394 len 40960 data: 10 11 11 10 9 12 10 12 11 15
186702024-01-10T18:02:56.709Z03/30 Read block 78 len 8192
186712024-01-10T18:02:56.709Z04/30 Read block 507 len 36864
186722024-01-10T18:02:56.715Z05/30 Read block 214 len 32768
186732024-01-10T18:02:56.715Z06/30 Read block 175 len 12288
186742024-01-10T18:02:56.715Z07/30 Write block 562 len 36864 data: 13 14 14 14 11 13 14 13 12
186752024-01-10T18:02:56.715Z08/30 Read block 369 len 32768
186762024-01-10T18:02:56.715Z09/30 Write block 587 len 24576 data: 12 9 10 9 9 8
186772024-01-10T18:02:56.715Z10/30 Write block 163 len 16384 data: 16 18 20 21
186782024-01-10T18:02:56.715Z11/30 Write block 99 len 20480 data: 15 14 9 9 8
186792024-01-10T18:02:56.715Z12/30 Read block 404 len 36864
186802024-01-10T18:02:56.716Z13/30 Flush
186812024-01-10T18:02:56.716Z14/30 Write block 590 len 20480 data: 10 10 9 10 13
186822024-01-10T18:02:56.716Z15/30 Flush
186832024-01-10T18:02:56.716Z16/30 Write block 430 len 28672 data: 12 11 11 13 12 13 13
186842024-01-10T18:02:56.717Z17/30 Read block 243 len 8192
186852024-01-10T18:02:56.717Z18/30 Write block 282 len 40960 data: 13 14 16 18 15 16 13 15 16 17
186862024-01-10T18:02:56.721Z19/30 Read block 184 len 20480
186872024-01-10T18:02:56.721Z20/30 Write block 500 len 16384 data: 15 14 12 13
186882024-01-10T18:02:56.721Z21/30 Write block 430 len 4096 data: 13
186892024-01-10T18:02:56.721Z22/30 Read block 554 len 28672
186902024-01-10T18:02:56.721Z23/30 Read block 216 len 24576
186912024-01-10T18:02:56.787Z24/30 Write block 385 len 40960 data: 10 9 9 8 10 11 11 12 9 11
186922024-01-10T18:02:56.798Z25/30 Write block 126 len 24576 data: 8 8 7 7 9 8
186932024-01-10T18:02:56.798Z26/30 Flush
186942024-01-10T18:02:56.798Z27/30 Write block 118 len 24576 data: 14 16 14 15 15 12
186952024-01-10T18:02:56.798Z28/30 Read block 539 len 36864
186962024-01-10T18:02:56.798Z29/30 Write block 10 len 36864 data: 12 12 9 9 12 13 14 15 16
186972024-01-10T18:02:56.798Z30/30 Write block 278 len 28672 data: 15 14 15 15 14 15 17
186982024-01-10T18:02:56.798Z----------------------------------------------------------------
186992024-01-10T18:02:56.798Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
187002024-01-10T18:02:56.798ZStates: Active Active Active
187012024-01-10T18:02:56.799ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187022024-01-10T18:02:56.799Z 1 Acked 1000 Write 8 Done Done Done false
187032024-01-10T18:02:56.799Z 2 Acked 1001 Write 10 Done Done Done false
187042024-01-10T18:02:56.799Z 3 Acked 1002 Read 2 Done Done Done false
187052024-01-10T18:02:56.801Z 4 Acked 1003 Read 9 Done Done Done false
187062024-01-10T18:02:56.801Z 5 Acked 1004 Read 8 Done Done Done false
187072024-01-10T18:02:56.801Z 6 Acked 1005 Read 3 Done Done Done false
187082024-01-10T18:02:56.801Z 7 Acked 1006 Write 9 Done Sent Done false
187092024-01-10T18:02:56.801Z 8 Acked 1007 Read 8 Done Sent Done false
187102024-01-10T18:02:56.801Z 9 Acked 1008 Write 6 Done Sent Done false
187112024-01-10T18:02:56.801Z 10 Acked 1009 Write 4 Done Sent Done false
187122024-01-10T18:02:56.801Z 11 Acked 1010 Write 5 Done Sent Done false
187132024-01-10T18:02:56.802Z 12 Acked 1011 Read 9 Done Sent Done false
187142024-01-10T18:02:56.802Z 13 Acked 1012 Flush 0 Done Sent Done false
187152024-01-10T18:02:56.802Z 14 Acked 1013 Write 5 Done Sent Done false
187162024-01-10T18:02:56.802Z 15 Acked 1014 Flush 0 Done Sent Done false
187172024-01-10T18:02:56.802Z 16 Acked 1015 Write 7 Done Sent Done false
187182024-01-10T18:02:56.802Z 17 Acked 1016 Read 2 Done Sent Done false
187192024-01-10T18:02:56.802Z 18 Acked 1017 Write 10 Done Sent Done false
187202024-01-10T18:02:56.802Z 19 Acked 1018 Read 5 Done Sent Done false
187212024-01-10T18:02:56.802Z 20 Acked 1019 Write 4 Done Sent Done false
187222024-01-10T18:02:56.802Z 21 Acked 1020 Write 1 Done Sent Done false
187232024-01-10T18:02:56.802Z 22 Acked 1021 Read 7 Done Sent Done false
187242024-01-10T18:02:56.803Z 23 Acked 1022 Read 6 Done Sent Done false
187252024-01-10T18:02:56.803Z 24 Acked 1023 Write 10 Done Sent Done false
187262024-01-10T18:02:56.803Z 25 Acked 1024 Write 6 Done Sent Done false
187272024-01-10T18:02:56.803Z 26 Acked 1025 Flush 0 Done Sent Done false
187282024-01-10T18:02:56.803Z 27 Acked 1026 Write 6 Done Sent Done false
187292024-01-10T18:02:56.803Z 28 Acked 1027 Read 9 Done Sent Done false
187302024-01-10T18:02:56.803Z 29 Acked 1028 Write 9 Sent Sent Sent false
187312024-01-10T18:02:56.803Z 30 Acked 1029 Write 7 Sent Sent Sent false
187322024-01-10T18:02:56.803Z STATES DS:0 DS:1 DS:2 TOTAL
187332024-01-10T18:02:56.803Z New 0 0 0 0
187342024-01-10T18:02:56.803Z Sent 2 24 2 28
187352024-01-10T18:02:56.803Z Done 28 6 28 62
187362024-01-10T18:02:56.804Z Skipped 0 0 0 0
187372024-01-10T18:02:56.804Z Error 0 0 0 0
187382024-01-10T18:02:56.804ZLast Flush: 1025 0 1025
187392024-01-10T18:02:56.804ZDownstairs last five completed:
187402024-01-10T18:02:56.804ZUpstairs last five completed: 30 29 28 27 26
187412024-01-10T18:02:56.804ZWrote out file "/tmp/test_repair_verify.data"
187422024-01-10T18:02:56.804Z
187432024-01-10T18:02:56.804ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
187442024-01-10T18:02:56.804ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187452024-01-10T18:02:56.804Z 0 000-019 172 172 172 316 316 316 T F T <---
187462024-01-10T18:02:56.805Z 1 020-039 172 172 172 316 316 316 F F F
187472024-01-10T18:02:56.805Z 2 040-059 172 172 172 314 314 314 F F F
187482024-01-10T18:02:56.806Z 3 060-079 172 172 172 314 314 314 F F F
187492024-01-10T18:02:56.806Z 4 080-099 174 172 174 319 317 319 F F F <---
187502024-01-10T18:02:56.806Z 5 100-119 174 172 174 319 317 319 T F T <---
187512024-01-10T18:02:56.806Z 6 120-139 174 160 174 321 285 321 T F T <---
187522024-01-10T18:02:56.806Z 7 140-159 172 172 172 316 316 316 F F F
187532024-01-10T18:02:56.808Z 8 160-179 174 168 174 319 306 319 F F F <---
187542024-01-10T18:02:56.808Z 9 180-199 173 173 173 318 318 318 F F F
187552024-01-10T18:02:56.808Z 10 200-219 173 173 173 318 318 318 F F F
187562024-01-10T18:02:56.808Z 11 220-239 174 168 174 319 305 319 F T F <---
187572024-01-10T18:02:56.808Z 12 240-259 166 166 166 300 300 300 F F F
187582024-01-10T18:02:56.808Z 13 260-279 172 172 172 316 316 316 T F T <---
187592024-01-10T18:02:56.808Z 14 280-299 174 172 174 321 315 321 T F T <---
187602024-01-10T18:02:56.809Z 15 300-319 169 169 169 307 307 307 F F F
187612024-01-10T18:02:56.809Z 16 320-339 155 155 155 274 274 274 F F F
187622024-01-10T18:02:56.809Z 17 340-359 167 167 167 303 303 303 F F F
187632024-01-10T18:02:56.809Z 18 360-379 171 171 171 313 313 313 F F F
187642024-01-10T18:02:56.810Z 19 380-399 174 172 174 321 316 321 F T F <---
187652024-01-10T18:02:56.810Z 20 400-419 174 168 174 319 305 319 F T F <---
187662024-01-10T18:02:56.810Z 21 420-439 174 173 174 321 318 321 F F F <---
187672024-01-10T18:02:56.810Z 22 440-459 159 159 159 283 283 283 F F F
187682024-01-10T18:02:56.810Z 23 460-479 173 173 173 318 318 318 F F F
187692024-01-10T18:02:56.810Z 24 480-499 170 170 170 309 309 309 F F F
187702024-01-10T18:02:56.810Z 25 500-519 174 170 174 321 309 321 F F F <---
187712024-01-10T18:02:56.811Z 26 520-539 172 172 172 314 314 314 F F F
187722024-01-10T18:02:56.811Z 27 540-559 172 172 172 314 314 314 F F F
187732024-01-10T18:02:56.811Z 28 560-579 174 172 174 319 315 319 F F F <---
187742024-01-10T18:02:56.811Z 29 580-599 174 164 174 320 295 320 F F F <---
187752024-01-10T18:02:56.811ZMax gen: 174, Max flush: 321
187762024-01-10T18:02:56.811ZError: Difference in extent metadata found!
187772024-01-10T18:02:56.811ZA Difference in extent metadata is expected here
187782024-01-10T18:02:56.811ZOn loop 87
187792024-01-10T18:02:56.811Z
187802024-01-10T18:02:56.811ZVerifying data now
187812024-01-10T18:03:02.148ZNo change after verify
187822024-01-10T18:03:02.148ZLoop: 87 Downstairs dump after verify (and repair):
187832024-01-10T18:03:02.170ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187842024-01-10T18:03:02.170Z 0 000-019 175 175 175 322 322 322 F F F
187852024-01-10T18:03:02.170Z 1 020-039 172 172 172 316 316 316 F F F
187862024-01-10T18:03:02.170Z 2 040-059 172 172 172 314 314 314 F F F
187872024-01-10T18:03:02.170Z 3 060-079 172 172 172 314 314 314 F F F
187882024-01-10T18:03:02.170Z 4 080-099 174 174 174 319 319 319 F F F
187892024-01-10T18:03:02.184Z 5 100-119 175 175 175 322 322 322 F F F
187902024-01-10T18:03:02.184Z 6 120-139 175 175 175 322 322 322 F F F
187912024-01-10T18:03:02.184Z 7 140-159 172 172 172 316 316 316 F F F
187922024-01-10T18:03:02.184Z 8 160-179 174 174 174 319 319 319 F F F
187932024-01-10T18:03:02.184Z 9 180-199 173 173 173 318 318 318 F F F
187942024-01-10T18:03:02.184Z 10 200-219 173 173 173 318 318 318 F F F
187952024-01-10T18:03:02.184Z 11 220-239 174 174 174 319 319 319 F F F
187962024-01-10T18:03:02.184Z 12 240-259 166 166 166 300 300 300 F F F
187972024-01-10T18:03:02.184Z 13 260-279 175 175 175 322 322 322 F F F
187982024-01-10T18:03:02.184Z 14 280-299 175 175 175 322 322 322 F F F
187992024-01-10T18:03:02.184Z 15 300-319 169 169 169 307 307 307 F F F
188002024-01-10T18:03:02.185Z 16 320-339 155 155 155 274 274 274 F F F
188012024-01-10T18:03:02.185Z 17 340-359 167 167 167 303 303 303 F F F
188022024-01-10T18:03:02.185Z 18 360-379 171 171 171 313 313 313 F F F
188032024-01-10T18:03:02.185Z 19 380-399 174 174 174 321 321 321 F F F
188042024-01-10T18:03:02.185Z 20 400-419 174 174 174 319 319 319 F F F
188052024-01-10T18:03:02.185Z 21 420-439 174 174 174 321 321 321 F F F
188062024-01-10T18:03:02.185Z 22 440-459 159 159 159 283 283 283 F F F
188072024-01-10T18:03:02.186Z 23 460-479 173 173 173 318 318 318 F F F
188082024-01-10T18:03:02.186Z 24 480-499 170 170 170 309 309 309 F F F
188092024-01-10T18:03:02.186Z 25 500-519 174 174 174 321 321 321 F F F
188102024-01-10T18:03:02.186Z 26 520-539 172 172 172 314 314 314 F F F
188112024-01-10T18:03:02.186Z 27 540-559 172 172 172 314 314 314 F F F
188122024-01-10T18:03:02.186Z 28 560-579 174 174 174 319 319 319 F F F
188132024-01-10T18:03:02.192Z 29 580-599 174 174 174 320 320 320 F F F
188142024-01-10T18:03:02.192ZMax gen: 175, Max flush: 322
188152024-01-10T18:03:02.192Z
188162024-01-10T18:03:02.192ZBegin loop 88
188172024-01-10T18:03:02.192ZDownstairs to restart: 1
188182024-01-10T18:03:02.608Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.202412596Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862}
188192024-01-10T18:03:02.609Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.202950955Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862}
188202024-01-10T18:03:02.609Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203007972Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862}
188212024-01-10T18:03:02.609Z{"msg":"Crucible bf38f396-b470-4bcb-8c1a-4afe3056fa50 has session id: fcea4903-f149-4bac-9329-fdd911b34d06","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203031149Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188222024-01-10T18:03:02.609Z{"msg":"Upstairs opts: Upstairs UUID: bf38f396-b470-4bcb-8c1a-4afe3056fa50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203072252Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188232024-01-10T18:03:02.613Z{"msg":"Crucible stats registered with UUID: bf38f396-b470-4bcb-8c1a-4afe3056fa50","v":0,"name":"crucible","level":30Crucible runtime is spawned
188242024-01-10T18:03:02.613Z,"time":"2024-01-10T18:03:02.203146375Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188252024-01-10T18:03:02.613Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203312022Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862}
188262024-01-10T18:03:02.613Z{"msg":"bf38f396-b470-4bcb-8c1a-4afe3056fa50 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203531566Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188272024-01-10T18:03:02.614Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203589459Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188282024-01-10T18:03:02.614Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203618226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188292024-01-10T18:03:02.614Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203674499Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188302024-01-10T18:03:02.614Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203710333Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188312024-01-10T18:03:02.614Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203750842Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188322024-01-10T18:03:02.614Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203810452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188332024-01-10T18:03:02.619Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203839498Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188342024-01-10T18:03:02.619Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203938758Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188352024-01-10T18:03:02.619Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.203983968Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188362024-01-10T18:03:02.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.204208762Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188372024-01-10T18:03:02.620Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.204327712Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188382024-01-10T18:03:02.620Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.204373366Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188392024-01-10T18:03:02.621Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.205502186Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188402024-01-10T18:03:02.621Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.205616021Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188412024-01-10T18:03:02.621Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.205707696Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188422024-01-10T18:03:02.621Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.205749037Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188432024-01-10T18:03:02.622Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.206412018Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188442024-01-10T18:03:02.622Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.206528686Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188452024-01-10T18:03:02.622Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.206628411Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188462024-01-10T18:03:02.622Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.206806881Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188472024-01-10T18:03:02.622Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.207611522Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188482024-01-10T18:03:02.622Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.207684625Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188492024-01-10T18:03:02.622Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.20771336Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188502024-01-10T18:03:02.622Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.207755757Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188512024-01-10T18:03:02.622Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.207780555Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188522024-01-10T18:03:02.622Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.207812283Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188532024-01-10T18:03:02.623Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.207856129Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188542024-01-10T18:03:02.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.208608294Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188552024-01-10T18:03:02.623Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.208694814Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188562024-01-10T18:03:02.623Z{"msg":"[0]R flush_numbers[0..12]: [322, 316, 314, 314, 319, 322, 322, 316, 319, 318, 318, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.208771831Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188572024-01-10T18:03:02.623Z{"msg":"[0]R generation[0..12]: [175, 172, 172, 172, 174, 175, 175, 172, 174, 173, 173, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.20884681Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188582024-01-10T18:03:02.623Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.20892423Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188592024-01-10T18:03:02.623Z{"msg":"[1]R flush_numbers[0..12]: [322, 316, 314, 314, 319, 322, 322, 316, 319, 318, 318, 319]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
188602024-01-10T18:03:02.624Z,"time":"2024-01-10T18:03:02.209034355Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188612024-01-10T18:03:02.624Z{"msg":"[1]R generation[0..12]: [175, 172, 172, 172, 174, 175, 175, 172, 174, 173, 173, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209166341Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188622024-01-10T18:03:02.624Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209250048Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188632024-01-10T18:03:02.645Z{"msg":"[2]R flush_numbers[0..12]: [322, 316, 314, 314, 319, 322, 322, 316, 319, 318, 318, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.2093266Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188642024-01-10T18:03:02.646Z{"msg":"[2]R generation[0..12]: [175, 172, 172, 172, 174, 175, 175, 172, 174, 173, 173, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209407704Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188652024-01-10T18:03:02.646Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209490903Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188662024-01-10T18:03:02.646Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209570492Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188672024-01-10T18:03:02.646Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
188682024-01-10T18:03:02.646Z,"time":"2024-01-10T18:03:02.209650318Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188692024-01-10T18:03:02.646Z{"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209725216Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188702024-01-10T18:03:02.647Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209775804Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188712024-01-10T18:03:02.647Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209799968Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188722024-01-10T18:03:02.647Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209838134Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188732024-01-10T18:03:02.648Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209861467Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188742024-01-10T18:03:02.648Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209901877Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188752024-01-10T18:03:02.648Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209925336Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188762024-01-10T18:03:02.648Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209945422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188772024-01-10T18:03:02.648Z{"msg":"bf38f396-b470-4bcb-8c1a-4afe3056fa50 is now active with session: fcea4903-f149-4bac-9329-fdd911b34d06","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.209984961Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188782024-01-10T18:03:02.648Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.210006113Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862,"session_id":"fcea4903-f149-4bac-9329-fdd911b34d06"}
188792024-01-10T18:03:02.649Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:02.210044698Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1862}
188802024-01-10T18:03:02.649ZLoading write count information from file "/tmp/test_repair_verify.data"
188812024-01-10T18:03:02.649ZRun Repair workload
188822024-01-10T18:03:02.649Z01/30 Write block 142 len 28672 data: 15 13 13 13 14 15 14
188832024-01-10T18:03:02.649Z02/30 Read block 482 len 36864
188842024-01-10T18:03:02.649Z03/30 Read block 354 len 4096
188852024-01-10T18:03:02.649Z04/30 Write block 295 len 28672 data: 16 14 18 21 17 14 13
188862024-01-10T18:03:02.650Z05/30 Read block 159 len 32768
188872024-01-10T18:03:02.650Z06/30 Read block 104 len 36864
188882024-01-10T18:03:02.676Z07/30 Read block 51 len 40960
188892024-01-10T18:03:02.676Z08/30 Write block 207 len 32768 data: 12 13 12 11 13 11 11 12
188902024-01-10T18:03:02.676Z09/30 Write block 44 len 32768 data: 11 11 9 7 12 14 15 13
188912024-01-10T18:03:02.676Z10/30 Read block 405 len 32768
188922024-01-10T18:03:02.676Z11/30 Write block 257 len 28672 data: 11 15 15 14 13 11 12
188932024-01-10T18:03:02.676Z12/30 Read block 163 len 8192
188942024-01-10T18:03:02.676Z13/30 Write block 181 len 20480 data: 12 12 11 10 12
188952024-01-10T18:03:02.676Z14/30 Flush
188962024-01-10T18:03:02.676Z15/30 Write block 323 len 20480 data: 12 14 14 12 11
188972024-01-10T18:03:02.677Z16/30 Write block 488 len 40960 data: 12 13 14 13 17 17 17 20 19 18
188982024-01-10T18:03:02.677Z17/30 Write block 559 len 32768 data: 10 11 13 14 15 15 15 12
188992024-01-10T18:03:02.677Z18/30 Read block 553 len 16384
189002024-01-10T18:03:02.677Z19/30 Read block 47 len 40960
189012024-01-10T18:03:02.677Z20/30 Read block 44 len 24576
189022024-01-10T18:03:02.677Z21/30 Write block 420 len 16384 data: 22 21 20 18
189032024-01-10T18:03:02.677Z22/30 Read block 463 len 28672
189042024-01-10T18:03:02.677Z23/30 Write block 572 len 40960 data: 12 12 16 15 15 14 13 14 17 15
189052024-01-10T18:03:02.677Z24/30 Read block 310 len 32768
189062024-01-10T18:03:02.677Z25/30 Write block 359 len 32768 data: 13 10 10 12 13 12 11 11
189072024-01-10T18:03:02.677Z26/30 Read block 93 len 36864
189082024-01-10T18:03:02.677Z27/30 Write block 431 len 32768 data: 12 12 14 13 14 14 14 14
189092024-01-10T18:03:02.677Z28/30 Write block 77 len 36864 data: 14 10 12 12 12 11 14 10 10
189102024-01-10T18:03:02.677Z29/30 Read block 374 len 12288
189112024-01-10T18:03:02.677Z30/30 Write block 75 len 32768 data: 15 15 15 11 13 13 13 12
189122024-01-10T18:03:02.677Z----------------------------------------------------------------
189132024-01-10T18:03:02.679Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
189142024-01-10T18:03:02.679ZStates: Active Active Active
189152024-01-10T18:03:02.679ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189162024-01-10T18:03:02.679Z 1 Acked 1000 Write 7 Done Sent Done false
189172024-01-10T18:03:02.679Z 2 Acked 1001 Read 9 Done Sent Done false
189182024-01-10T18:03:02.722Z 3 Acked 1002 Read 1 Done Sent Done false
189192024-01-10T18:03:02.722Z 4 Acked 1003 Write 7 Done Sent Done false
189202024-01-10T18:03:02.722Z 5 Acked 1004 Read 8 Done Sent Done false
189212024-01-10T18:03:02.723Z 6 Acked 1005 Read 9 Done Sent Done false
189222024-01-10T18:03:02.723Z 7 Acked 1006 Read 10 Done Sent Done false
189232024-01-10T18:03:02.723Z 8 Acked 1007 Write 8 Done Sent Done false
189242024-01-10T18:03:02.723Z 9 Acked 1008 Write 8 Done Sent Done false
189252024-01-10T18:03:02.723Z 10 Acked 1009 Read 8 Done Sent Done false
189262024-01-10T18:03:02.723Z 11 Acked 1010 Write 7 Done Sent Done false
189272024-01-10T18:03:02.723Z 12 Acked 1011 Read 2 Done Sent Done false
189282024-01-10T18:03:02.723Z 13 Acked 1012 Write 5 Done Sent Done false
189292024-01-10T18:03:02.723Z 14 Acked 1013 Flush 0 Done Sent Done false
189302024-01-10T18:03:02.723Z 15 Acked 1014 Write 5 Done Sent Done false
189312024-01-10T18:03:02.723Z 16 Acked 1015 Write 10 Done Sent Done false
189322024-01-10T18:03:02.728Z 17 Acked 1016 Write 8 Done Sent Done false
189332024-01-10T18:03:02.728Z 18 Acked 1017 Read 4 Done Sent Done false
189342024-01-10T18:03:02.728Z 19 Acked 1018 Read 10 Done Sent Done false
189352024-01-10T18:03:02.728Z 20 Acked 1019 Read 6 Done Sent Done false
189362024-01-10T18:03:02.728Z 21 Acked 1020 Write 4 Done Sent Done false
189372024-01-10T18:03:02.728Z 22 Acked 1021 Read 7 Done Sent Done false
189382024-01-10T18:03:02.728Z 23 Acked 1022 Write 10 Done Sent Done false
189392024-01-10T18:03:02.728Z 24 Acked 1023 Read 8 Done Sent Done false
189402024-01-10T18:03:02.728Z 25 Acked 1024 Write 8 Done Sent Done false
189412024-01-10T18:03:02.728Z 26 Acked 1025 Read 9 Done Sent Done false
189422024-01-10T18:03:02.728Z 27 Acked 1026 Write 8 Done Sent Done false
189432024-01-10T18:03:02.728Z 28 Acked 1027 Write 9 Done Sent Done false
189442024-01-10T18:03:02.729Z 29 Acked 1028 Read 3 Done Sent Done false
189452024-01-10T18:03:02.729Z 30 Acked 1029 Write 8 Sent Sent Sent false
189462024-01-10T18:03:02.729Z STATES DS:0 DS:1 DS:2 TOTAL
189472024-01-10T18:03:02.729Z New 0 0 0 0
189482024-01-10T18:03:02.729Z Sent 1 30 1 32
189492024-01-10T18:03:02.729Z Done 29 0 29 58
189502024-01-10T18:03:02.729Z Skipped 0 0 0 0
189512024-01-10T18:03:02.729Z Error 0 0 0 0
189522024-01-10T18:03:02.729ZLast Flush: 1013 0 1013
189532024-01-10T18:03:02.729ZDownstairs last five completed:
189542024-01-10T18:03:02.729ZUpstairs last five completed: 30 29 28 27 26
189552024-01-10T18:03:02.729ZWrote out file "/tmp/test_repair_verify.data"
189562024-01-10T18:03:02.729Z
189572024-01-10T18:03:02.729ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
189582024-01-10T18:03:02.730ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189592024-01-10T18:03:02.730Z 0 000-019 175 175 175 322 322 322 F F F
189602024-01-10T18:03:02.730Z 1 020-039 172 172 172 316 316 316 F F F
189612024-01-10T18:03:02.730Z 2 040-059 176 172 176 323 314 323 F F F <---
189622024-01-10T18:03:02.730Z 3 060-079 172 172 172 314 314 314 T F T <---
189632024-01-10T18:03:02.730Z 4 080-099 174 174 174 319 319 319 T F T <---
189642024-01-10T18:03:02.730Z 5 100-119 175 175 175 322 322 322 F F F
189652024-01-10T18:03:02.730Z 6 120-139 175 175 175 322 322 322 F F F
189662024-01-10T18:03:02.730Z 7 140-159 176 172 176 323 316 323 F F F <---
189672024-01-10T18:03:02.730Z 8 160-179 174 174 174 319 319 319 F F F
189682024-01-10T18:03:02.730Z 9 180-199 176 173 176 323 318 323 F F F <---
189692024-01-10T18:03:02.730Z 10 200-219 176 173 176 323 318 323 F F F <---
189702024-01-10T18:03:02.730Z 11 220-239 174 174 174 319 319 319 F F F
189712024-01-10T18:03:02.730Z 12 240-259 176 166 176 323 300 323 F F F <---
189722024-01-10T18:03:02.730Z 13 260-279 176 175 176 323 322 323 F F F <---
189732024-01-10T18:03:02.730Z 14 280-299 176 175 176 323 322 323 F F F <---
189742024-01-10T18:03:02.731Z 15 300-319 176 169 176 323 307 323 F F F <---
189752024-01-10T18:03:02.731Z 16 320-339 155 155 155 274 274 274 T F T <---
189762024-01-10T18:03:02.731Z 17 340-359 167 167 167 303 303 303 T F T <---
189772024-01-10T18:03:02.731Z 18 360-379 171 171 171 313 313 313 T F T <---
189782024-01-10T18:03:02.731Z 19 380-399 174 174 174 321 321 321 F F F
189792024-01-10T18:03:02.731Z 20 400-419 174 174 174 319 319 319 F F F
189802024-01-10T18:03:02.731Z 21 420-439 174 174 174 321 321 321 T F T <---
189812024-01-10T18:03:02.731Z 22 440-459 159 159 159 283 283 283 F F F
189822024-01-10T18:03:02.731Z 23 460-479 173 173 173 318 318 318 F F F
189832024-01-10T18:03:02.731Z 24 480-499 170 170 170 309 309 309 T F T <---
189842024-01-10T18:03:02.731Z 25 500-519 174 174 174 321 321 321 F F F
189852024-01-10T18:03:02.731Z 26 520-539 172 172 172 314 314 314 F F F
189862024-01-10T18:03:02.732Z 27 540-559 172 172 172 314 314 314 T F T <---
189872024-01-10T18:03:02.732Z 28 560-579 174 174 174 319 319 319 T F T <---
189882024-01-10T18:03:02.732Z 29 580-599 174 174 174 320 320 320 T F T <---
189892024-01-10T18:03:02.732ZMax gen: 176, Max flush: 323
189902024-01-10T18:03:02.732ZError: Difference in extent metadata found!
189912024-01-10T18:03:02.732ZA Difference in extent metadata is expected here
189922024-01-10T18:03:02.732ZOn loop 88
189932024-01-10T18:03:02.732Z
189942024-01-10T18:03:02.732ZVerifying data now
189952024-01-10T18:03:07.988ZNo change after verify
189962024-01-10T18:03:07.988ZLoop: 88 Downstairs dump after verify (and repair):
189972024-01-10T18:03:08.010ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189982024-01-10T18:03:08.010Z 0 000-019 175 175 175 322 322 322 F F F
189992024-01-10T18:03:08.010Z 1 020-039 172 172 172 316 316 316 F F F
190002024-01-10T18:03:08.010Z 2 040-059 176 176 176 323 323 323 F F F
190012024-01-10T18:03:08.015Z 3 060-079 177 177 177 324 324 324 F F F
190022024-01-10T18:03:08.015Z 4 080-099 177 177 177 324 324 324 F F F
190032024-01-10T18:03:08.018Z 5 100-119 175 175 175 322 322 322 F F F
190042024-01-10T18:03:08.019Z 6 120-139 175 175 175 322 322 322 F F F
190052024-01-10T18:03:08.019Z 7 140-159 176 176 176 323 323 323 F F F
190062024-01-10T18:03:08.019Z 8 160-179 174 174 174 319 319 319 F F F
190072024-01-10T18:03:08.019Z 9 180-199 176 176 176 323 323 323 F F F
190082024-01-10T18:03:08.019Z 10 200-219 176 176 176 323 323 323 F F F
190092024-01-10T18:03:08.019Z 11 220-239 174 174 174 319 319 319 F F F
190102024-01-10T18:03:08.019Z 12 240-259 176 176 176 323 323 323 F F F
190112024-01-10T18:03:08.019Z 13 260-279 176 176 176 323 323 323 F F F
190122024-01-10T18:03:08.019Z 14 280-299 176 176 176 323 323 323 F F F
190132024-01-10T18:03:08.019Z 15 300-319 176 176 176 323 323 323 F F F
190142024-01-10T18:03:08.019Z 16 320-339 177 177 177 324 324 324 F F F
190152024-01-10T18:03:08.019Z 17 340-359 177 177 177 324 324 324 F F F
190162024-01-10T18:03:08.019Z 18 360-379 177 177 177 324 324 324 F F F
190172024-01-10T18:03:08.019Z 19 380-399 174 174 174 321 321 321 F F F
190182024-01-10T18:03:08.019Z 20 400-419 174 174 174 319 319 319 F F F
190192024-01-10T18:03:08.019Z 21 420-439 177 177 177 324 324 324 F F F
190202024-01-10T18:03:08.019Z 22 440-459 159 159 159 283 283 283 F F F
190212024-01-10T18:03:08.019Z 23 460-479 173 173 173 318 318 318 F F F
190222024-01-10T18:03:08.019Z 24 480-499 177 177 177 324 324 324 F F F
190232024-01-10T18:03:08.019Z 25 500-519 174 174 174 321 321 321 F F F
190242024-01-10T18:03:08.019Z 26 520-539 172 172 172 314 314 314 F F F
190252024-01-10T18:03:08.020Z 27 540-559 177 177 177 324 324 324 F F F
190262024-01-10T18:03:08.020Z 28 560-579 177 177 177 324 324 324 F F F
190272024-01-10T18:03:08.020Z 29 580-599 177 177 177 324 324 324 F F F
190282024-01-10T18:03:08.020ZMax gen: 177, Max flush: 324
190292024-01-10T18:03:08.020Z
190302024-01-10T18:03:08.020ZBegin loop 89
190312024-01-10T18:03:08.020ZDownstairs to restart: 1
190322024-01-10T18:03:08.044Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.041312252Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870}
190332024-01-10T18:03:08.044Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.041819817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870}
190342024-01-10T18:03:08.044Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.04184941Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870}
190352024-01-10T18:03:08.050Z{"msg":"Crucible 52db5a6c-8ea4-410d-aca7-7ead3926c0a5 has session id: a7899b41-27f3-4cb2-b4ea-a0dd5d58f018","v":0,"name":"crucible","level":30Crucible runtime is spawned
190362024-01-10T18:03:08.050Z,"time":"2024-01-10T18:03:08.041906167Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190372024-01-10T18:03:08.050Z{"msg":"Upstairs opts: Upstairs UUID: 52db5a6c-8ea4-410d-aca7-7ead3926c0a5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042065682Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190382024-01-10T18:03:08.050Z{"msg":"Crucible stats registered with UUID: 52db5a6c-8ea4-410d-aca7-7ead3926c0a5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042098577Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190392024-01-10T18:03:08.050Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042136912Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870}
190402024-01-10T18:03:08.051Z{"msg":"52db5a6c-8ea4-410d-aca7-7ead3926c0a5 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042296966Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190412024-01-10T18:03:08.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042364018Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190422024-01-10T18:03:08.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042409219Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190432024-01-10T18:03:08.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042451137Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190442024-01-10T18:03:08.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042563044Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190452024-01-10T18:03:08.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042589134Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190462024-01-10T18:03:08.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042695416Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190472024-01-10T18:03:08.052Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042742403Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190482024-01-10T18:03:08.052Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042819534Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190492024-01-10T18:03:08.052Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.042868456Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190502024-01-10T18:03:08.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.043080985Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190512024-01-10T18:03:08.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.043165098Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190522024-01-10T18:03:08.054Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.043209379Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190532024-01-10T18:03:08.054Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.044030438Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190542024-01-10T18:03:08.054Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.044103853Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190552024-01-10T18:03:08.054Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.044131172Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190562024-01-10T18:03:08.054Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.044153735Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190572024-01-10T18:03:08.054Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.044773891Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190582024-01-10T18:03:08.054Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.044882835Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190592024-01-10T18:03:08.054Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.045052973Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190602024-01-10T18:03:08.054Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.0452965Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190612024-01-10T18:03:08.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.045848511Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190622024-01-10T18:03:08.054Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.045972581Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190632024-01-10T18:03:08.054Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.04602112Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190642024-01-10T18:03:08.054Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.046096283Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190652024-01-10T18:03:08.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.046205439Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190662024-01-10T18:03:08.054Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.04623828Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190672024-01-10T18:03:08.054Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.046280345Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190682024-01-10T18:03:08.058Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.046805562Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190692024-01-10T18:03:08.058Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.04686537Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190702024-01-10T18:03:08.059Z{"msg":"[0]R flush_numbers[0..12]: [322, 316, 323, 324, 324, 322, 322, 323, 319, 323, 323, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.046910309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190712024-01-10T18:03:08.059Z{"msg":"[0]R generation[0..12]: [175, 172, 176, 177, 177, 175, 175, 176, 174, 176, 176, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.046951498Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190722024-01-10T18:03:08.059Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047045473Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190732024-01-10T18:03:08.059Z{"msg":"[1]R flush_numbers[0..12]: [322, 316, 323, 324, 324, 322, 322, 323, 319, 323, 323, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047111964Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190742024-01-10T18:03:08.059Z{"msg":"[1]R generation[0..12]: [175, 172, 176, 177, 177, 175, 175, 176, 174, 176, 176, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047140346Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190752024-01-10T18:03:08.059ZWait for a query_work_queue command to finish before sending IO
190762024-01-10T18:03:08.059Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047205247Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190772024-01-10T18:03:08.059Z{"msg":"[2]R flush_numbers[0..12]: [322, 316, 323, 324, 324, 322, 322, 323, 319, 323, 323, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047271009Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190782024-01-10T18:03:08.059Z{"msg":"[2]R generation[0..12]: [175, 172, 176, 177, 177, 175, 175, 176, 174, 176, 176, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047299923Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190792024-01-10T18:03:08.059Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.0473239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190802024-01-10T18:03:08.059Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047385207Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190812024-01-10T18:03:08.059Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047412253Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190822024-01-10T18:03:08.059Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047475923Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190832024-01-10T18:03:08.060Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047535871Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190842024-01-10T18:03:08.060Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047562361Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190852024-01-10T18:03:08.060Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047624897Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190862024-01-10T18:03:08.060Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047651395Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190872024-01-10T18:03:08.060Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047674192Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190882024-01-10T18:03:08.060Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047798078Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190892024-01-10T18:03:08.060ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
190902024-01-10T18:03:08.060Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.04787338Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190912024-01-10T18:03:08.064Z{"msg":"52db5a6c-8ea4-410d-aca7-7ead3926c0a5 is now active with session: a7899b41-27f3-4cb2-b4ea-a0dd5d58f018","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.047945139Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190922024-01-10T18:03:08.064Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.048011723Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870,"session_id":"a7899b41-27f3-4cb2-b4ea-a0dd5d58f018"}
190932024-01-10T18:03:08.064Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:08.04807773Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1870}
190942024-01-10T18:03:08.064ZLoading write count information from file "/tmp/test_repair_verify.data"
190952024-01-10T18:03:08.064ZRun Repair workload
190962024-01-10T18:03:08.064Z01/30 Write block 452 len 32768 data: 14 17 14 12 13 13 11 11
190972024-01-10T18:03:08.064Z02/30 Read block 98 len 20480
190982024-01-10T18:03:08.064Z03/30 Read block 436 len 24576
190992024-01-10T18:03:08.068Z04/30 Write block 208 len 8192 data: 14 13
191002024-01-10T18:03:08.068Z05/30 Write block 179 len 24576 data: 12 15 13 13 12 11
191012024-01-10T18:03:08.068Z06/30 Read block 421 len 20480
191022024-01-10T18:03:08.068Z07/30 Read block 266 len 36864
191032024-01-10T18:03:08.068Z08/30 Flush
191042024-01-10T18:03:08.073Z09/30 Write block 65 len 28672 data: 16 18 17 14 12 13 9
191052024-01-10T18:03:08.073Z10/30 Write block 95 len 12288 data: 16 17 17
191062024-01-10T18:03:08.078Z11/30 Read block 187 len 32768
191072024-01-10T18:03:08.078Z12/30 Write block 552 len 28672 data: 21 21 21 17 16 14 11
191082024-01-10T18:03:08.078Z13/30 Read block 166 len 24576
191092024-01-10T18:03:08.088Z14/30 Read block 424 len 36864
191102024-01-10T18:03:08.088Z15/30 Write block 442 len 16384 data: 11 11 10 14
191112024-01-10T18:03:08.099Z16/30 Write block 306 len 20480 data: 10 8 6 6 6
191122024-01-10T18:03:08.100Z17/30 Flush
191132024-01-10T18:03:08.100Z18/30 Write block 77 len 40960 data: 16 12 14 14 14 13 15 11 11 9
191142024-01-10T18:03:08.100Z19/30 Read block 306 len 12288
191152024-01-10T18:03:08.101Z20/30 Flush
191162024-01-10T18:03:08.108Z21/30 Write block 284 len 24576 data: 18 19 16 17 14 16
191172024-01-10T18:03:08.108Z22/30 Read block 94 len 4096
191182024-01-10T18:03:08.119Z23/30 Read block 329 len 24576
191192024-01-10T18:03:08.119Z24/30 Flush
191202024-01-10T18:03:08.120Z25/30 Write block 461 len 36864 data: 10 10 13 13 11 15 16 16 13
191212024-01-10T18:03:08.120Z26/30 Read block 249 len 16384
191222024-01-10T18:03:08.120Z27/30 Read block 570 len 40960
191232024-01-10T18:03:08.130Z28/30 Read block 268 len 32768
191242024-01-10T18:03:08.130Z29/30 Write block 122 len 32768 data: 16 13 10 10 9 9 8 8
191252024-01-10T18:03:08.130Z30/30 Write block 38 len 36864 data: 11 9 8 11 12 10 12 12 10
191262024-01-10T18:03:08.130Z----------------------------------------------------------------
191272024-01-10T18:03:08.130Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
191282024-01-10T18:03:08.130ZStates: Active Active Active
191292024-01-10T18:03:08.130ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
191302024-01-10T18:03:08.130Z 1 Acked 1000 Write 8 Done Done Done false
191312024-01-10T18:03:08.130Z 2 Acked 1001 Read 5 Done Done Done false
191322024-01-10T18:03:08.130Z 3 Acked 1002 Read 6 Done Sent Done false
191332024-01-10T18:03:08.130Z 4 Acked 1003 Write 2 Done Sent Done false
191342024-01-10T18:03:08.130Z 5 Acked 1004 Write 6 Done Sent Done false
191352024-01-10T18:03:08.130Z 6 Acked 1005 Read 5 Done Sent Done false
191362024-01-10T18:03:08.130Z 7 Acked 1006 Read 9 Done Sent Done false
191372024-01-10T18:03:08.131Z 8 Acked 1007 Flush 0 Done Sent Done false
191382024-01-10T18:03:08.131Z 9 Acked 1008 Write 7 Done Sent Done false
191392024-01-10T18:03:08.131Z 10 Acked 1009 Write 3 Done Sent Done false
191402024-01-10T18:03:08.131Z 11 Acked 1010 Read 8 Done Sent Done false
191412024-01-10T18:03:08.131Z 12 Acked 1011 Write 7 Done Sent Done false
191422024-01-10T18:03:08.131Z 13 Acked 1012 Read 6 Done Sent Done false
191432024-01-10T18:03:08.131Z 14 Acked 1013 Read 9 Done Sent Done false
191442024-01-10T18:03:08.131Z 15 Acked 1014 Write 4 Done Sent Done false
191452024-01-10T18:03:08.131Z 16 Acked 1015 Write 5 Done Sent Done false
191462024-01-10T18:03:08.131Z 17 Acked 1016 Flush 0 Done Sent Done false
191472024-01-10T18:03:08.131Z 18 Acked 1017 Write 10 Done Sent Done false
191482024-01-10T18:03:08.131Z 19 Acked 1018 Read 3 Done Sent Done false
191492024-01-10T18:03:08.131Z 20 Acked 1019 Flush 0 Done Sent Done false
191502024-01-10T18:03:08.131Z 21 Acked 1020 Write 6 Done Sent Done false
191512024-01-10T18:03:08.132Z 22 Acked 1021 Read 1 Done Sent Done false
191522024-01-10T18:03:08.132Z 23 Acked 1022 Read 6 Done Sent Done false
191532024-01-10T18:03:08.132Z 24 Acked 1023 Flush 0 Done Sent Done false
191542024-01-10T18:03:08.132Z 25 Acked 1024 Write 9 Done Sent Done false
191552024-01-10T18:03:08.132Z 26 Acked 1025 Read 4 Done Sent Done false
191562024-01-10T18:03:08.132Z 27 Acked 1026 Read 10 Done Sent Done false
191572024-01-10T18:03:08.132Z 28 Acked 1027 Read 8 Done Sent Done false
191582024-01-10T18:03:08.132Z 29 Acked 1028 Write 8 Done Sent Done false
191592024-01-10T18:03:08.132Z 30 Acked 1029 Write 9 Sent Sent Sent false
191602024-01-10T18:03:08.132Z STATES DS:0 DS:1 DS:2 TOTAL
191612024-01-10T18:03:08.132Z New 0 0 0 0
191622024-01-10T18:03:08.132Z Sent 1 28 1 30
191632024-01-10T18:03:08.132Z Done 29 2 29 60
191642024-01-10T18:03:08.132Z Skipped 0 0 0 0
191652024-01-10T18:03:08.132Z Error 0 0 0 0
191662024-01-10T18:03:08.132ZLast Flush: 1023 0 1023
191672024-01-10T18:03:08.132ZDownstairs last five completed:
191682024-01-10T18:03:08.133ZUpstairs last five completed: 30 29 28 27 26
191692024-01-10T18:03:08.133ZWrote out file "/tmp/test_repair_verify.data"
191702024-01-10T18:03:08.133Z
191712024-01-10T18:03:08.139ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
191722024-01-10T18:03:08.156ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191732024-01-10T18:03:08.157Z 0 000-019 175 175 175 322 322 322 F F F
191742024-01-10T18:03:08.157Z 1 020-039 172 172 172 316 316 316 T F T <---
191752024-01-10T18:03:08.157Z 2 040-059 176 176 176 323 323 323 T F T <---
191762024-01-10T18:03:08.167Z 3 060-079 178 177 178 327 324 327 F F F <---
191772024-01-10T18:03:08.167Z 4 080-099 178 177 178 327 324 327 F F F <---
191782024-01-10T18:03:08.167Z 5 100-119 175 175 175 322 322 322 F F F
191792024-01-10T18:03:08.167Z 6 120-139 175 175 175 322 322 322 T F T <---
191802024-01-10T18:03:08.168Z 7 140-159 176 176 176 323 323 323 F F F
191812024-01-10T18:03:08.168Z 8 160-179 178 174 178 325 319 325 F F F <---
191822024-01-10T18:03:08.168Z 9 180-199 178 176 178 325 323 325 F F F <---
191832024-01-10T18:03:08.169Z 10 200-219 178 176 178 325 323 325 F F F <---
191842024-01-10T18:03:08.169Z 11 220-239 174 174 174 319 319 319 F F F
191852024-01-10T18:03:08.170Z 12 240-259 176 176 176 323 323 323 F F F
191862024-01-10T18:03:08.170Z 13 260-279 176 176 176 323 323 323 F F F
191872024-01-10T18:03:08.170Z 14 280-299 178 176 178 328 323 328 F F F <---
191882024-01-10T18:03:08.170Z 15 300-319 178 176 178 326 323 326 F F F <---
191892024-01-10T18:03:08.170Z 16 320-339 177 177 177 324 324 324 F F F
191902024-01-10T18:03:08.170Z 17 340-359 177 177 177 324 324 324 F F F
191912024-01-10T18:03:08.171Z 18 360-379 177 177 177 324 324 324 F F F
191922024-01-10T18:03:08.171Z 19 380-399 174 174 174 321 321 321 F F F
191932024-01-10T18:03:08.171Z 20 400-419 174 174 174 319 319 319 F F F
191942024-01-10T18:03:08.171Z 21 420-439 177 177 177 324 324 324 F F F
191952024-01-10T18:03:08.171Z 22 440-459 178 159 178 326 283 326 F T F <---
191962024-01-10T18:03:08.171Z 23 460-479 173 173 173 318 318 318 T F T <---
191972024-01-10T18:03:08.171Z 24 480-499 177 177 177 324 324 324 F F F
191982024-01-10T18:03:08.171Z 25 500-519 174 174 174 321 321 321 F F F
191992024-01-10T18:03:08.172Z 26 520-539 172 172 172 314 314 314 F F F
192002024-01-10T18:03:08.172Z 27 540-559 178 177 178 326 324 326 F F F <---
192012024-01-10T18:03:08.172Z 28 560-579 177 177 177 324 324 324 F F F
192022024-01-10T18:03:08.172Z 29 580-599 177 177 177 324 324 324 F F F
192032024-01-10T18:03:08.172ZMax gen: 178, Max flush: 328
192042024-01-10T18:03:08.172ZError: Difference in extent metadata found!
192052024-01-10T18:03:08.172ZA Difference in extent metadata is expected here
192062024-01-10T18:03:08.173ZOn loop 89
192072024-01-10T18:03:08.173Z
192082024-01-10T18:03:08.173ZVerifying data now
192092024-01-10T18:03:13.420ZNo change after verify
192102024-01-10T18:03:13.420ZLoop: 89 Downstairs dump after verify (and repair):
192112024-01-10T18:03:13.442ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192122024-01-10T18:03:13.442Z 0 000-019 175 175 175 322 322 322 F F F
192132024-01-10T18:03:13.443Z 1 020-039 179 179 179 329 329 329 F F F
192142024-01-10T18:03:13.443Z 2 040-059 179 179 179 329 329 329 F F F
192152024-01-10T18:03:13.443Z 3 060-079 178 178 178 327 327 327 F F F
192162024-01-10T18:03:13.443Z 4 080-099 178 178 178 327 327 327 F F F
192172024-01-10T18:03:13.443Z 5 100-119 175 175 175 322 322 322 F F F
192182024-01-10T18:03:13.443Z 6 120-139 179 179 179 329 329 329 F F F
192192024-01-10T18:03:13.446Z 7 140-159 176 176 176 323 323 323 F F F
192202024-01-10T18:03:13.446Z 8 160-179 178 178 178 325 325 325 F F F
192212024-01-10T18:03:13.446Z 9 180-199 178 178 178 325 325 325 F F F
192222024-01-10T18:03:13.446Z 10 200-219 178 178 178 325 325 325 F F F
192232024-01-10T18:03:13.446Z 11 220-239 174 174 174 319 319 319 F F F
192242024-01-10T18:03:13.446Z 12 240-259 176 176 176 323 323 323 F F F
192252024-01-10T18:03:13.446Z 13 260-279 176 176 176 323 323 323 F F F
192262024-01-10T18:03:13.446Z 14 280-299 178 178 178 328 328 328 F F F
192272024-01-10T18:03:13.446Z 15 300-319 178 178 178 326 326 326 F F F
192282024-01-10T18:03:13.446Z 16 320-339 177 177 177 324 324 324 F F F
192292024-01-10T18:03:13.446Z 17 340-359 177 177 177 324 324 324 F F F
192302024-01-10T18:03:13.446Z 18 360-379 177 177 177 324 324 324 F F F
192312024-01-10T18:03:13.446Z 19 380-399 174 174 174 321 321 321 F F F
192322024-01-10T18:03:13.446Z 20 400-419 174 174 174 319 319 319 F F F
192332024-01-10T18:03:13.446Z 21 420-439 177 177 177 324 324 324 F F F
192342024-01-10T18:03:13.446Z 22 440-459 178 178 178 326 326 326 F F F
192352024-01-10T18:03:13.446Z 23 460-479 179 179 179 329 329 329 F F F
192362024-01-10T18:03:13.446Z 24 480-499 177 177 177 324 324 324 F F F
192372024-01-10T18:03:13.447Z 25 500-519 174 174 174 321 321 321 F F F
192382024-01-10T18:03:13.447Z 26 520-539 172 172 172 314 314 314 F F F
192392024-01-10T18:03:13.447Z 27 540-559 178 178 178 326 326 326 F F F
192402024-01-10T18:03:13.447Z 28 560-579 177 177 177 324 324 324 F F F
192412024-01-10T18:03:13.447Z 29 580-599 177 177 177 324 324 324 F F F
192422024-01-10T18:03:13.447ZMax gen: 179, Max flush: 329
192432024-01-10T18:03:13.450Z
192442024-01-10T18:03:13.450ZBegin loop 90
192452024-01-10T18:03:13.450ZDownstairs to restart: 2
192462024-01-10T18:03:13.480Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.476699622Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878}
192472024-01-10T18:03:13.480Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.477281143Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878}
192482024-01-10T18:03:13.480Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"Crucible runtime is spawned
192492024-01-10T18:03:13.480Zcrucible","level":30,"time":"2024-01-10T18:03:13.477359143Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878}
192502024-01-10T18:03:13.485Z{"msg":"Crucible 014ddf64-cfdd-4983-a362-f129293bbdf3 has session id: 9592fbe4-f889-4efa-aa4f-fbf19a58cc28","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.477460437Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192512024-01-10T18:03:13.485Z{"msg":"Upstairs opts: Upstairs UUID: 014ddf64-cfdd-4983-a362-f129293bbdf3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.477586022Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192522024-01-10T18:03:13.485Z{"msg":"Crucible stats registered with UUID: 014ddf64-cfdd-4983-a362-f129293bbdf3","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.477657002Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192532024-01-10T18:03:13.485Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.477721862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878}
192542024-01-10T18:03:13.485Z{"msg":"014ddf64-cfdd-4983-a362-f129293bbdf3 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.477756168Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192552024-01-10T18:03:13.485Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.477817448Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192562024-01-10T18:03:13.485Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.477909656Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192572024-01-10T18:03:13.485Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.478008032Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192582024-01-10T18:03:13.485Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.478056683Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192592024-01-10T18:03:13.485Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.478131753Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192602024-01-10T18:03:13.485Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.478205257Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192612024-01-10T18:03:13.485Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.478247054Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192622024-01-10T18:03:13.485Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.478333332Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192632024-01-10T18:03:13.485Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.478374097Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192642024-01-10T18:03:13.485Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.478411382Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192652024-01-10T18:03:13.485Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.478448944Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192662024-01-10T18:03:13.485Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.478514812Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192672024-01-10T18:03:13.485Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.479472099Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192682024-01-10T18:03:13.485Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.479569358Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192692024-01-10T18:03:13.485Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.479614199Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192702024-01-10T18:03:13.485Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.479658129Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192712024-01-10T18:03:13.485Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.48034642Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192722024-01-10T18:03:13.486Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.480559655Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192732024-01-10T18:03:13.486Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.480854937Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192742024-01-10T18:03:13.486Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.480921439Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192752024-01-10T18:03:13.490Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.481190713Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192762024-01-10T18:03:13.490Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.481249576Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192772024-01-10T18:03:13.490Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.481298563Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192782024-01-10T18:03:13.490Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.481338203Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192792024-01-10T18:03:13.491Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.48136488Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192802024-01-10T18:03:13.491Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.481387755Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192812024-01-10T18:03:13.491Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.481775393Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192822024-01-10T18:03:13.493Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.482534799Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192832024-01-10T18:03:13.493Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.482706541Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192842024-01-10T18:03:13.493Z{"msg":"[0]R flush_numbers[0..12]: [322, 329, 329, 327, 327, 322, 329, 323, 325, 325, 325, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.482772539Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192852024-01-10T18:03:13.493Z{"msg":"[0]R generation[0..12]: [175, 179, 179, 178, 178, 175, 179, 176, 178, 178, 178, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.482799381Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"Wait for a query_work_queue command to finish before sending IO
192862024-01-10T18:03:13.493Zdownstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192872024-01-10T18:03:13.493Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.482893026Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192882024-01-10T18:03:13.493Z{"msg":"[1]R flush_numbers[0..12]: [322, 329, 329, 327, 327, 322, 329, 323, 325, 325, 325, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.482941938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192892024-01-10T18:03:13.493Z{"msg":"[1]R generation[0..12]: [175, 179, 179, 178, 178, 175, 179, 176, 178, 178, 178, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.482967066Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192902024-01-10T18:03:13.493Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483022153Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192912024-01-10T18:03:13.493Z{"msg":"[2]R flush_numbers[0..12]: [322, 329, 329, 327, 327, 322, 329, 323, 325, 325, 325, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483047178Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192922024-01-10T18:03:13.493Z{"msg":"[2]R generation[0..12]: [175, 179, 179, 178, 178, 175, 179, 176, 178, 178, 178, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483070315Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192932024-01-10T18:03:13.493Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.48311118Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192942024-01-10T18:03:13.493Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483134742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192952024-01-10T18:03:13.494Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483157027Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192962024-01-10T18:03:13.494Z{"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483232308Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192972024-01-10T18:03:13.494Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.48325535Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192982024-01-10T18:03:13.494Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483277237Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
192992024-01-10T18:03:13.494Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.4833298Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
193002024-01-10T18:03:13.494Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483353356Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
193012024-01-10T18:03:13.494Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483411327Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
193022024-01-10T18:03:13.494Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483440379Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
193032024-01-10T18:03:13.494Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483461519Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
193042024-01-10T18:03:13.494Z{"msg":"014ddf64-cfdd-4983-a362-f129293bbdf3 is now active with session: 9592fbe4-f889-4efa-aa4f-fbf19a58cc28","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483483163Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
193052024-01-10T18:03:13.494Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483522169Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878,"session_id":"9592fbe4-f889-4efa-aa4f-fbf19a58cc28"}
193062024-01-10T18:03:13.494Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:13.483543859Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1878}
193072024-01-10T18:03:13.494ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193082024-01-10T18:03:13.494ZLoading write count information from file "/tmp/test_repair_verify.data"
193092024-01-10T18:03:13.494ZRun Repair workload
193102024-01-10T18:03:13.494Z01/30 Flush
193112024-01-10T18:03:13.494Z02/30 Write block 452 len 40960 data: 15 18 15 13 14 14 12 12 12 11
193122024-01-10T18:03:13.498Z03/30 Read block 566 len 40960
193132024-01-10T18:03:13.498Z04/30 Read block 70 len 24576
193142024-01-10T18:03:13.498Z05/30 Read block 322 len 40960
193152024-01-10T18:03:13.533Z06/30 Read block 248 len 28672
193162024-01-10T18:03:13.537Z07/30 Flush
193172024-01-10T18:03:13.537Z08/30 Write block 573 len 24576 data: 13 17 16 16 15 14
193182024-01-10T18:03:13.537Z09/30 Write block 138 len 8192 data: 15 15
193192024-01-10T18:03:13.537Z10/30 Write block 556 len 8192 data: 17 15
193202024-01-10T18:03:13.537Z11/30 Flush
193212024-01-10T18:03:13.537Z12/30 Flush
193222024-01-10T18:03:13.537Z13/30 Write block 22 len 4096 data: 16
193232024-01-10T18:03:13.537Z14/30 Read block 322 len 24576
193242024-01-10T18:03:13.537Z15/30 Write block 47 len 32768 data: 8 13 15 16 14 13 10 15
193252024-01-10T18:03:13.537Z16/30 Read block 514 len 32768
193262024-01-10T18:03:13.537Z17/30 Flush
193272024-01-10T18:03:13.537Z18/30 Write block 372 len 28672 data: 11 11 10 11 9 9 8
193282024-01-10T18:03:13.537Z19/30 Read block 72 len 40960
193292024-01-10T18:03:13.537Z20/30 Read block 347 len 16384
193302024-01-10T18:03:13.537Z21/30 Read block 90 len 36864
193312024-01-10T18:03:13.537Z22/30 Write block 254 len 36864 data: 7 7 10 12 16 16 15 14 12
193322024-01-10T18:03:13.537Z23/30 Read block 516 len 36864
193332024-01-10T18:03:13.541Z24/30 Write block 511 len 8192 data: 17 17
193342024-01-10T18:03:13.541Z25/30 Write block 352 len 28672 data: 13 10 11 9 11 12 15
193352024-01-10T18:03:13.541Z26/30 Read block 281 len 20480
193362024-01-10T18:03:13.543Z27/30 Write block 531 len 12288 data: 19 16 17
193372024-01-10T18:03:13.544Z28/30 Write block 105 len 12288 data: 7 8 7
193382024-01-10T18:03:13.547Z29/30 Write block 427 len 4096 data: 15
193392024-01-10T18:03:13.547Z30/30 Write block 1 len 28672 data: 7 7 8 10 9 9 8
193402024-01-10T18:03:13.548Z----------------------------------------------------------------
193412024-01-10T18:03:13.548Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:29
193422024-01-10T18:03:13.548ZStates: Active Active Active
193432024-01-10T18:03:13.548ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
193442024-01-10T18:03:13.548Z 2 Acked 1001 Write 10 Done Done Done false
193452024-01-10T18:03:13.549Z 3 Acked 1002 Read 10 Done Done Done false
193462024-01-10T18:03:13.549Z 4 Acked 1003 Read 6 Done Done Sent false
193472024-01-10T18:03:13.549Z 5 Acked 1004 Read 10 Done Done Sent false
193482024-01-10T18:03:13.549Z 6 Acked 1005 Read 7 Done Done Sent false
193492024-01-10T18:03:13.549Z 7 Acked 1006 Flush 0 Done Done Sent false
193502024-01-10T18:03:13.549Z 8 Acked 1007 Write 6 Done Done Sent false
193512024-01-10T18:03:13.549Z 9 Acked 1008 Write 2 Done Done Sent false
193522024-01-10T18:03:13.549Z 10 Acked 1009 Write 2 Done Done Sent false
193532024-01-10T18:03:13.549Z 11 Acked 1010 Flush 0 Done Done Sent false
193542024-01-10T18:03:13.550Z 12 Acked 1011 Flush 0 Done Done Sent false
193552024-01-10T18:03:13.550Z 13 Acked 1012 Write 1 Done Done Sent false
193562024-01-10T18:03:13.550Z 14 Acked 1013 Read 6 Done Done Sent false
193572024-01-10T18:03:13.550Z 15 Acked 1014 Write 8 Done Done Sent false
193582024-01-10T18:03:13.550Z 16 Acked 1015 Read 8 Done Done Sent false
193592024-01-10T18:03:13.550Z 17 Acked 1016 Flush 0 Done Done Sent false
193602024-01-10T18:03:13.550Z 18 Acked 1017 Write 7 Done Done Sent false
193612024-01-10T18:03:13.550Z 19 Acked 1018 Read 10 Done Done Sent false
193622024-01-10T18:03:13.550Z 20 Acked 1019 Read 4 Done Done Sent false
193632024-01-10T18:03:13.550Z 21 Acked 1020 Read 9 Done Done Sent false
193642024-01-10T18:03:13.550Z 22 Acked 1021 Write 9 Done Done Sent false
193652024-01-10T18:03:13.550Z 23 Acked 1022 Read 9 Done Done Sent false
193662024-01-10T18:03:13.550Z 24 Acked 1023 Write 2 Done Done Sent false
193672024-01-10T18:03:13.550Z 25 Acked 1024 Write 7 Done Done Sent false
193682024-01-10T18:03:13.550Z 26 Acked 1025 Read 5 Done Done Sent false
193692024-01-10T18:03:13.550Z 27 Acked 1026 Write 3 Done Done Sent false
193702024-01-10T18:03:13.550Z 28 Acked 1027 Write 3 Done Done Sent false
193712024-01-10T18:03:13.550Z 29 Acked 1028 Write 1 Done Done Sent false
193722024-01-10T18:03:13.550Z 30 Acked 1029 Write 7 Sent Sent Sent false
193732024-01-10T18:03:13.550Z STATES DS:0 DS:1 DS:2 TOTAL
193742024-01-10T18:03:13.550Z New 0 0 0 0
193752024-01-10T18:03:13.550Z Sent 1 1 27 29
193762024-01-10T18:03:13.550Z Done 28 28 2 58
193772024-01-10T18:03:13.550Z Skipped 0 0 0 0
193782024-01-10T18:03:13.550Z Error 0 0 0 0
193792024-01-10T18:03:13.551ZLast Flush: 1016 1016 1000
193802024-01-10T18:03:13.551ZDownstairs last five completed: 1000
193812024-01-10T18:03:13.551ZUpstairs last five completed: 30 29 28 27 26
193822024-01-10T18:03:13.554ZWrote out file "/tmp/test_repair_verify.data"
193832024-01-10T18:03:13.564Z
193842024-01-10T18:03:13.564ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
193852024-01-10T18:03:13.590ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193862024-01-10T18:03:13.590Z 0 000-019 175 175 175 322 322 322 T T F <---
193872024-01-10T18:03:13.590Z 1 020-039 180 180 179 334 334 329 F F F <---
193882024-01-10T18:03:13.590Z 2 040-059 180 180 179 334 334 329 F F F <---
193892024-01-10T18:03:13.590Z 3 060-079 178 178 178 327 327 327 F F F
193902024-01-10T18:03:13.590Z 4 080-099 178 178 178 327 327 327 F F F
193912024-01-10T18:03:13.590Z 5 100-119 175 175 175 322 322 322 T T F <---
193922024-01-10T18:03:13.590Z 6 120-139 180 180 179 332 332 329 F F F <---
193932024-01-10T18:03:13.590Z 7 140-159 176 176 176 323 323 323 F F F
193942024-01-10T18:03:13.590Z 8 160-179 178 178 178 325 325 325 F F F
193952024-01-10T18:03:13.590Z 9 180-199 178 178 178 325 325 325 F F F
193962024-01-10T18:03:13.590Z 10 200-219 178 178 178 325 325 325 F F F
193972024-01-10T18:03:13.590Z 11 220-239 174 174 174 319 319 319 F F F
193982024-01-10T18:03:13.594Z 12 240-259 176 176 176 323 323 323 T T F <---
193992024-01-10T18:03:13.594Z 13 260-279 176 176 176 323 323 323 T T F <---
194002024-01-10T18:03:13.594Z 14 280-299 178 178 178 328 328 328 F F F
194012024-01-10T18:03:13.594Z 15 300-319 178 178 178 326 326 326 F F F
194022024-01-10T18:03:13.594Z 16 320-339 177 177 177 324 324 324 F F F
194032024-01-10T18:03:13.594Z 17 340-359 177 177 177 324 324 324 T T F <---
194042024-01-10T18:03:13.594Z 18 360-379 177 177 177 324 324 324 T T F <---
194052024-01-10T18:03:13.594Z 19 380-399 174 174 174 321 321 321 F F F
194062024-01-10T18:03:13.594Z 20 400-419 174 174 174 319 319 319 F F F
194072024-01-10T18:03:13.594Z 21 420-439 177 177 177 324 324 324 T T F <---
194082024-01-10T18:03:13.594Z 22 440-459 180 180 178 331 331 326 F F T <---
194092024-01-10T18:03:13.594Z 23 460-479 180 180 179 331 331 329 F F T <---
194102024-01-10T18:03:13.594Z 24 480-499 177 177 177 324 324 324 F F F
194112024-01-10T18:03:13.594Z 25 500-519 174 174 174 321 321 321 T T F <---
194122024-01-10T18:03:13.594Z 26 520-539 172 172 172 314 314 314 T T F <---
194132024-01-10T18:03:13.594Z 27 540-559 180 180 178 332 332 326 F F F <---
194142024-01-10T18:03:13.594Z 28 560-579 180 180 177 332 332 324 F F F <---
194152024-01-10T18:03:13.594Z 29 580-599 177 177 177 324 324 324 F F F
194162024-01-10T18:03:13.594ZMax gen: 180, Max flush: 334
194172024-01-10T18:03:13.594ZError: Difference in extent metadata found!
194182024-01-10T18:03:13.597ZA Difference in extent metadata is expected here
194192024-01-10T18:03:13.597ZOn loop 90
194202024-01-10T18:03:13.597Z
194212024-01-10T18:03:13.601ZVerifying data now
194222024-01-10T18:03:15.142ZNo change after verify
194232024-01-10T18:03:15.142ZLoop: 90 Downstairs dump after verify (and repair):
194242024-01-10T18:03:15.164ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194252024-01-10T18:03:15.164Z 0 000-019 181 181 181 335 335 335 F F F
194262024-01-10T18:03:15.164Z 1 020-039 180 180 180 334 334 334 F F F
194272024-01-10T18:03:15.164Z 2 040-059 180 180 180 334 334 334 F F F
194282024-01-10T18:03:15.164Z 3 060-079 178 178 178 327 327 327 F F F
194292024-01-10T18:03:15.164Z 4 080-099 178 178 178 327 327 327 F F F
194302024-01-10T18:03:15.164Z 5 100-119 181 181 181 335 335 335 F F F
194312024-01-10T18:03:15.167Z 6 120-139 180 180 180 332 332 332 F F F
194322024-01-10T18:03:15.167Z 7 140-159 176 176 176 323 323 323 F F F
194332024-01-10T18:03:15.168Z 8 160-179 178 178 178 325 325 325 F F F
194342024-01-10T18:03:15.168Z 9 180-199 178 178 178 325 325 325 F F F
194352024-01-10T18:03:15.168Z 10 200-219 178 178 178 325 325 325 F F F
194362024-01-10T18:03:15.168Z 11 220-239 174 174 174 319 319 319 F F F
194372024-01-10T18:03:15.168Z 12 240-259 181 181 181 335 335 335 F F F
194382024-01-10T18:03:15.168Z 13 260-279 181 181 181 335 335 335 F F F
194392024-01-10T18:03:15.168Z 14 280-299 178 178 178 328 328 328 F F F
194402024-01-10T18:03:15.168Z 15 300-319 178 178 178 326 326 326 F F F
194412024-01-10T18:03:15.168Z 16 320-339 177 177 177 324 324 324 F F F
194422024-01-10T18:03:15.168Z 17 340-359 181 181 181 335 335 335 F F F
194432024-01-10T18:03:15.168Z 18 360-379 181 181 181 335 335 335 F F F
194442024-01-10T18:03:15.168Z 19 380-399 174 174 174 321 321 321 F F F
194452024-01-10T18:03:15.168Z 20 400-419 174 174 174 319 319 319 F F F
194462024-01-10T18:03:15.168Z 21 420-439 181 181 181 335 335 335 F F F
194472024-01-10T18:03:15.168Z 22 440-459 180 180 180 331 331 331 F F F
194482024-01-10T18:03:15.169Z 23 460-479 180 180 180 331 331 331 F F F
194492024-01-10T18:03:15.170Z 24 480-499 177 177 177 324 324 324 F F F
194502024-01-10T18:03:15.170Z 25 500-519 181 181 181 335 335 335 F F F
194512024-01-10T18:03:15.170Z 26 520-539 181 181 181 335 335 335 F F F
194522024-01-10T18:03:15.170Z 27 540-559 180 180 180 332 332 332 F F F
194532024-01-10T18:03:15.170Z 28 560-579 180 180 180 332 332 332 F F F
194542024-01-10T18:03:15.170Z 29 580-599 177 177 177 324 324 324 F F F
194552024-01-10T18:03:15.170ZMax gen: 181, Max flush: 335
194562024-01-10T18:03:15.173Z
194572024-01-10T18:03:15.173ZBegin loop 91
194582024-01-10T18:03:15.173ZDownstairs to restart: 0
194592024-01-10T18:03:15.199Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.196519424Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886}
194602024-01-10T18:03:15.204Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.197081179Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886}
194612024-01-10T18:03:15.204Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.197132092Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886}
194622024-01-10T18:03:15.204Z{"msg":"Crucible d84aa3f3-3bab-4733-b46a-3f92fb8a2f1b has session id: 5dfbf146-1fa8-4796-85b5-de1ee8004020","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.197158339Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194632024-01-10T18:03:15.204Z{"msg":"Upstairs opts: Upstairs UUID: d84aa3f3-3bab-4733-b46a-3f92fb8a2f1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.197213594Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194642024-01-10T18:03:15.204Z{"msg":"Crucible stats registered with UUID: d84aa3f3-3bab-4733-b46a-3f92fb8a2f1b","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.197257026Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194652024-01-10T18:03:15.204ZCrucible runtime is spawned
194662024-01-10T18:03:15.204Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.197580744Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886}
194672024-01-10T18:03:15.204Z{"msg":"d84aa3f3-3bab-4733-b46a-3f92fb8a2f1b active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.197769555Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194682024-01-10T18:03:15.204Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.197845808Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194692024-01-10T18:03:15.204Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.197926215Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194702024-01-10T18:03:15.204Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.197971518Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194712024-01-10T18:03:15.204Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.19804254Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194722024-01-10T18:03:15.204Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.198086197Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194732024-01-10T18:03:15.205Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.198123844Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194742024-01-10T18:03:15.205Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.198160619Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194752024-01-10T18:03:15.205Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.19820294Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194762024-01-10T18:03:15.205Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.198243469Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194772024-01-10T18:03:15.205Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.198285552Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194782024-01-10T18:03:15.205Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.198430116Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194792024-01-10T18:03:15.205Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.198485758Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194802024-01-10T18:03:15.205Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.199515197Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194812024-01-10T18:03:15.205Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.199591392Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194822024-01-10T18:03:15.205Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.199638436Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194832024-01-10T18:03:15.205Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.199742213Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194842024-01-10T18:03:15.205Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.200251874Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194852024-01-10T18:03:15.205Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.200350391Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194862024-01-10T18:03:15.210Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.200526333Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194872024-01-10T18:03:15.210Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.20087147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194882024-01-10T18:03:15.210Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.201693752Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194892024-01-10T18:03:15.210Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.201756348Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194902024-01-10T18:03:15.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.201782365Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194912024-01-10T18:03:15.212Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.201836341Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194922024-01-10T18:03:15.213Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.201939483Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194932024-01-10T18:03:15.213Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.201972155Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194942024-01-10T18:03:15.213Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.20203004Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194952024-01-10T18:03:15.213Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.202601902Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194962024-01-10T18:03:15.213Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.202690327Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194972024-01-10T18:03:15.213Z{"msg":"[0]R flush_numbers[0..12]: [335, 334, 334, 327, 327, 335, 332, 323, 325, 325, 325, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.202753782Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194982024-01-10T18:03:15.213Z{"msg":"[0]R generation[0..12]: [181, 180, 180, 178, 178, 181, 180, 176, 178, 178, 178, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.202781927Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
194992024-01-10T18:03:15.213Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.202832545Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195002024-01-10T18:03:15.213Z{"msg":"[1]R flush_numbers[0..12]: [335, 334, 334, 327, 327, 335, 332, 323, 325, 325, 325, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.202858023Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195012024-01-10T18:03:15.213Z{"msg":"[1]R generation[0..12]: [181, 180, 180, 178, 178, 181, 180, 176, 178, 178, 178, 174]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
195022024-01-10T18:03:15.213Z,"time":"2024-01-10T18:03:15.202906697Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195032024-01-10T18:03:15.213Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.202932473Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195042024-01-10T18:03:15.214Z{"msg":"[2]R flush_numbers[0..12]: [335, 334, 334, 327, 327, 335, 332, 323, 325, 325, 325, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.202955863Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195052024-01-10T18:03:15.214Z{"msg":"[2]R generation[0..12]: [181, 180, 180, 178, 178, 181, 180, 176, 178, 178, 178, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.202996936Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195062024-01-10T18:03:15.214Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203021737Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195072024-01-10T18:03:15.214Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.20306166Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195082024-01-10T18:03:15.214Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203085145Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195092024-01-10T18:03:15.214Z{"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203106222Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195102024-01-10T18:03:15.214Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203145299Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195112024-01-10T18:03:15.214Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203168389Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195122024-01-10T18:03:15.214Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203188293Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195132024-01-10T18:03:15.214Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203230037Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195142024-01-10T18:03:15.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203253053Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195152024-01-10T18:03:15.215Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203274753Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195162024-01-10T18:03:15.215Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203319455Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195172024-01-10T18:03:15.215Z{"msg":"d84aa3f3-3bab-4733-b46a-3f92fb8a2f1b is now active with session: 5dfbf146-1fa8-4796-85b5-de1ee8004020","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.20334235Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195182024-01-10T18:03:15.215Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203361676Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886,"session_id":"5dfbf146-1fa8-4796-85b5-de1ee8004020"}
195192024-01-10T18:03:15.215Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:15.203381905Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1886}
195202024-01-10T18:03:15.215ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
195212024-01-10T18:03:15.215ZLoading write count information from file "/tmp/test_repair_verify.data"
195222024-01-10T18:03:15.215ZRun Repair workload
195232024-01-10T18:03:15.218Z01/30 Write block 181 len 8192 data: 14 14
195242024-01-10T18:03:15.219Z02/30 Flush
195252024-01-10T18:03:15.219Z03/30 Write block 257 len 40960 data: 13 17 17 16 15 13 13 14 12 14
195262024-01-10T18:03:15.219Z04/30 Read block 533 len 16384
195272024-01-10T18:03:15.222Z05/30 Flush
195282024-01-10T18:03:15.222Z06/30 Write block 33 len 8192 data: 14 15
195292024-01-10T18:03:15.225Z07/30 Read block 302 len 28672
195302024-01-10T18:03:15.228Z08/30 Write block 483 len 20480 data: 10 12 9 12 11
195312024-01-10T18:03:15.229Z09/30 Write block 54 len 4096 data: 16
195322024-01-10T18:03:15.229Z10/30 Write block 295 len 36864 data: 17 15 19 22 18 15 14 14 14
195332024-01-10T18:03:15.229Z11/30 Flush
195342024-01-10T18:03:15.236Z12/30 Write block 509 len 28672 data: 16 15 18 18 16 19 18
195352024-01-10T18:03:15.239Z13/30 Read block 450 len 40960
195362024-01-10T18:03:15.239Z14/30 Write block 39 len 32768 data: 10 9 12 13 11 13 13 11
195372024-01-10T18:03:15.239Z15/30 Read block 140 len 20480
195382024-01-10T18:03:15.243Z16/30 Flush
195392024-01-10T18:03:15.245Z17/30 Write block 380 len 32768 data: 9 10 10 7 9 11 10 10
195402024-01-10T18:03:15.248Z18/30 Write block 134 len 20480 data: 10 11 9 14 16
195412024-01-10T18:03:15.248Z19/30 Write block 15 len 12288 data: 14 15 16
195422024-01-10T18:03:15.248Z20/30 Write block 516 len 12288 data: 15 14 13
195432024-01-10T18:03:15.248Z21/30 Read block 530 len 16384
195442024-01-10T18:03:15.260Z22/30 Write block 192 len 40960 data: 14 15 17 19 18 19 20 20 18 17
195452024-01-10T18:03:15.272Z23/30 Write block 171 len 32768 data: 12 12 11 13 10 9 11 11
195462024-01-10T18:03:15.272Z24/30 Read block 577 len 4096
195472024-01-10T18:03:15.272Z25/30 Flush
195482024-01-10T18:03:15.272Z26/30 Write block 331 len 8192 data: 9 7
195492024-01-10T18:03:15.272Z27/30 Write block 550 len 16384 data: 20 19 22 22
195502024-01-10T18:03:15.272Z28/30 Read block 326 len 12288
195512024-01-10T18:03:15.272Z29/30 Write block 17 len 24576 data: 17 17 15 15 16 17
195522024-01-10T18:03:15.272Z30/30 Read block 489 len 4096
195532024-01-10T18:03:15.703Z----------------------------------------------------------------
195542024-01-10T18:03:15.704Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
195552024-01-10T18:03:15.704ZStates: Active Active Active
195562024-01-10T18:03:15.704ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
195572024-01-10T18:03:15.704Z 1 Acked 1000 Write 2 Sent Done Done false
195582024-01-10T18:03:15.704Z 2 Acked 1001 Flush 0 Sent Done Done false
195592024-01-10T18:03:15.704Z 3 Acked 1002 Write 10 Sent Done Done false
195602024-01-10T18:03:15.704Z 4 Acked 1003 Read 4 Sent Done Done false
195612024-01-10T18:03:15.704Z 5 Acked 1004 Flush 0 Sent Done Done false
195622024-01-10T18:03:15.704Z 6 Acked 1005 Write 2 Sent Done Done false
195632024-01-10T18:03:15.704Z 7 Acked 1006 Read 7 Sent Done Done false
195642024-01-10T18:03:15.704Z 8 Acked 1007 Write 5 Sent Done Done false
195652024-01-10T18:03:15.704Z 9 Acked 1008 Write 1 Sent Done Done false
195662024-01-10T18:03:15.704Z 10 Acked 1009 Write 9 Sent Done Done false
195672024-01-10T18:03:15.705Z 11 Acked 1010 Flush 0 Sent Done Done false
195682024-01-10T18:03:15.705Z 12 Acked 1011 Write 7 Sent Done Done false
195692024-01-10T18:03:15.705Z 13 Acked 1012 Read 10 Sent Done Done false
195702024-01-10T18:03:15.705Z 14 Acked 1013 Write 8 Sent Done Done false
195712024-01-10T18:03:15.705Z 15 Acked 1014 Read 5 Sent Done Done false
195722024-01-10T18:03:15.705Z 16 Acked 1015 Flush 0 Sent Done Done false
195732024-01-10T18:03:15.705Z 17 Acked 1016 Write 8 Sent Done Done false
195742024-01-10T18:03:15.705Z 18 Acked 1017 Write 5 Sent Done Done false
195752024-01-10T18:03:15.707Z 19 Acked 1018 Write 3 Sent Done Done false
195762024-01-10T18:03:15.707Z 20 Acked 1019 Write 3 Sent Done Done false
195772024-01-10T18:03:15.707Z 21 Acked 1020 Read 4 Sent Done Done false
195782024-01-10T18:03:15.707Z 22 Acked 1021 Write 10 Sent Done Done false
195792024-01-10T18:03:15.707Z 23 Acked 1022 Write 8 Sent Done Done false
195802024-01-10T18:03:15.707Z 24 Acked 1023 Read 1 Sent Done Done false
195812024-01-10T18:03:15.707Z 25 Acked 1024 Flush 0 Sent Done Done false
195822024-01-10T18:03:15.707Z 26 Acked 1025 Write 2 Sent Done Done false
195832024-01-10T18:03:15.707Z 27 Acked 1026 Write 4 Sent Done Done false
195842024-01-10T18:03:15.707Z 28 Acked 1027 Read 3 Sent Done Done false
195852024-01-10T18:03:15.707Z 29 Acked 1028 Write 6 Sent Done Sent false
195862024-01-10T18:03:15.708Z 30 Acked 1029 Read 1 Sent Done Sent false
195872024-01-10T18:03:15.708Z STATES DS:0 DS:1 DS:2 TOTAL
195882024-01-10T18:03:15.708Z New 0 0 0 0
195892024-01-10T18:03:15.708Z Sent 30 0 2 32
195902024-01-10T18:03:15.708Z Done 0 30 28 58
195912024-01-10T18:03:15.708Z Skipped 0 0 0 0
195922024-01-10T18:03:15.708Z Error 0 0 0 0
195932024-01-10T18:03:15.708ZLast Flush: 0 1024 1024
195942024-01-10T18:03:15.708ZDownstairs last five completed:
195952024-01-10T18:03:15.708ZUpstairs last five completed: 30 29 28 27 26
195962024-01-10T18:03:15.708ZWrote out file "/tmp/test_repair_verify.data"
195972024-01-10T18:03:15.708Z
195982024-01-10T18:03:15.709ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
195992024-01-10T18:03:15.709ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196002024-01-10T18:03:15.709Z 0 000-019 181 182 182 335 340 340 F T T <---
196012024-01-10T18:03:15.709Z 1 020-039 180 182 182 334 339 339 F T T <---
196022024-01-10T18:03:15.709Z 2 040-059 180 182 182 334 339 339 F F F <---
196032024-01-10T18:03:15.709Z 3 060-079 178 178 178 327 327 327 F F F
196042024-01-10T18:03:15.709Z 4 080-099 178 178 178 327 327 327 F F F
196052024-01-10T18:03:15.709Z 5 100-119 181 181 181 335 335 335 F F F
196062024-01-10T18:03:15.709Z 6 120-139 180 182 182 332 340 340 F F F <---
196072024-01-10T18:03:15.709Z 7 140-159 176 176 176 323 323 323 F F F
196082024-01-10T18:03:15.709Z 8 160-179 178 182 182 325 340 340 F F F <---
196092024-01-10T18:03:15.709Z 9 180-199 178 182 182 325 340 340 F F F <---
196102024-01-10T18:03:15.709Z 10 200-219 178 182 182 325 340 340 F F F <---
196112024-01-10T18:03:15.709Z 11 220-239 174 174 174 319 319 319 F F F
196122024-01-10T18:03:15.709Z 12 240-259 181 182 182 335 337 337 F F F <---
196132024-01-10T18:03:15.709Z 13 260-279 181 182 182 335 337 337 F F F <---
196142024-01-10T18:03:15.709Z 14 280-299 178 182 182 328 338 338 F F F <---
196152024-01-10T18:03:15.710Z 15 300-319 178 182 182 326 338 338 F F F <---
196162024-01-10T18:03:15.710Z 16 320-339 177 177 177 324 324 324 F T T <---
196172024-01-10T18:03:15.710Z 17 340-359 181 181 181 335 335 335 F F F
196182024-01-10T18:03:15.710Z 18 360-379 181 181 181 335 335 335 F F F
196192024-01-10T18:03:15.710Z 19 380-399 174 182 182 321 340 340 F F F <---
196202024-01-10T18:03:15.710Z 20 400-419 174 174 174 319 319 319 F F F
196212024-01-10T18:03:15.710Z 21 420-439 181 181 181 335 335 335 F F F
196222024-01-10T18:03:15.711Z 22 440-459 180 180 180 331 331 331 F F F
196232024-01-10T18:03:15.711Z 23 460-479 180 180 180 331 331 331 F F F
196242024-01-10T18:03:15.711Z 24 480-499 177 182 182 324 338 338 F F F <---
196252024-01-10T18:03:15.711Z 25 500-519 181 182 182 335 340 340 F F F <---
196262024-01-10T18:03:15.711Z 26 520-539 181 181 181 335 335 335 F F F
196272024-01-10T18:03:15.711Z 27 540-559 180 180 180 332 332 332 F T T <---
196282024-01-10T18:03:15.711Z 28 560-579 180 180 180 332 332 332 F F F
196292024-01-10T18:03:15.711Z 29 580-599 177 177 177 324 324 324 F F F
196302024-01-10T18:03:15.711ZMax gen: 182, Max flush: 340
196312024-01-10T18:03:15.711ZError: Difference in extent metadata found!
196322024-01-10T18:03:15.711ZA Difference in extent metadata is expected here
196332024-01-10T18:03:15.711ZOn loop 91
196342024-01-10T18:03:15.712Z
196352024-01-10T18:03:15.712ZVerifying data now
196362024-01-10T18:03:20.829ZNo change after verify
196372024-01-10T18:03:20.829ZLoop: 91 Downstairs dump after verify (and repair):
196382024-01-10T18:03:20.851ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196392024-01-10T18:03:20.851Z 0 000-019 183 183 183 341 341 341 F F F
196402024-01-10T18:03:20.851Z 1 020-039 183 183 183 341 341 341 F F F
196412024-01-10T18:03:20.851Z 2 040-059 182 182 182 339 339 339 F F F
196422024-01-10T18:03:20.858Z 3 060-079 178 178 178 327 327 327 F F F
196432024-01-10T18:03:20.859Z 4 080-099 178 178 178 327 327 327 F F F
196442024-01-10T18:03:20.859Z 5 100-119 181 181 181 335 335 335 F F F
196452024-01-10T18:03:20.859Z 6 120-139 182 182 182 340 340 340 F F F
196462024-01-10T18:03:20.859Z 7 140-159 176 176 176 323 323 323 F F F
196472024-01-10T18:03:20.859Z 8 160-179 182 182 182 340 340 340 F F F
196482024-01-10T18:03:20.859Z 9 180-199 182 182 182 340 340 340 F F F
196492024-01-10T18:03:20.859Z 10 200-219 182 182 182 340 340 340 F F F
196502024-01-10T18:03:20.859Z 11 220-239 174 174 174 319 319 319 F F F
196512024-01-10T18:03:20.859Z 12 240-259 182 182 182 337 337 337 F F F
196522024-01-10T18:03:20.859Z 13 260-279 182 182 182 337 337 337 F F F
196532024-01-10T18:03:20.859Z 14 280-299 182 182 182 338 338 338 F F F
196542024-01-10T18:03:20.859Z 15 300-319 182 182 182 338 338 338 F F F
196552024-01-10T18:03:20.859Z 16 320-339 183 183 183 341 341 341 F F F
196562024-01-10T18:03:20.859Z 17 340-359 181 181 181 335 335 335 F F F
196572024-01-10T18:03:20.859Z 18 360-379 181 181 181 335 335 335 F F F
196582024-01-10T18:03:20.859Z 19 380-399 182 182 182 340 340 340 F F F
196592024-01-10T18:03:20.859Z 20 400-419 174 174 174 319 319 319 F F F
196602024-01-10T18:03:20.859Z 21 420-439 181 181 181 335 335 335 F F F
196612024-01-10T18:03:20.859Z 22 440-459 180 180 180 331 331 331 F F F
196622024-01-10T18:03:20.859Z 23 460-479 180 180 180 331 331 331 F F F
196632024-01-10T18:03:20.859Z 24 480-499 182 182 182 338 338 338 F F F
196642024-01-10T18:03:20.859Z 25 500-519 182 182 182 340 340 340 F F F
196652024-01-10T18:03:20.859Z 26 520-539 181 181 181 335 335 335 F F F
196662024-01-10T18:03:20.860Z 27 540-559 183 183 183 341 341 341 F F F
196672024-01-10T18:03:20.860Z 28 560-579 180 180 180 332 332 332 F F F
196682024-01-10T18:03:20.860Z 29 580-599 177 177 177 324 324 324 F F F
196692024-01-10T18:03:20.861ZMax gen: 183, Max flush: 341
196702024-01-10T18:03:20.861Z
196712024-01-10T18:03:20.861ZBegin loop 92
196722024-01-10T18:03:20.861ZDownstairs to restart: 0
196732024-01-10T18:03:20.885Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.882508889Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894}
196742024-01-10T18:03:20.891Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.882979356Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894}
196752024-01-10T18:03:20.891Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.883057545Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894}
196762024-01-10T18:03:20.892Z{"msg":"Crucible f599db80-3171-4c21-a2f3-15b402a71ab4 has session id: 2e1933e9-b3a9-4282-bd19-90c36a8e1f1c","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.883170522Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196772024-01-10T18:03:20.892Z{"msg":"Upstairs opts: Upstairs UUID: f599db80-3171-4c21-a2f3-15b402a71ab4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.88321196Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196782024-01-10T18:03:20.892Z{"msg":"Crucible stats registered with UUID: f599db80-3171-4c21-a2f3-15b402a71ab4","v":0,"name":"crucibleCrucible runtime is spawned
196792024-01-10T18:03:20.892Z","level":30,"time":"2024-01-10T18:03:20.883376946Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196802024-01-10T18:03:20.892Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.883514155Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894}
196812024-01-10T18:03:20.893Z{"msg":"f599db80-3171-4c21-a2f3-15b402a71ab4 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.883668596Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196822024-01-10T18:03:20.893Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.883737504Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196832024-01-10T18:03:20.893Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.883780387Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196842024-01-10T18:03:20.893Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.883903307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196852024-01-10T18:03:20.893Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.883949938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196862024-01-10T18:03:20.894Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.884153575Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196872024-01-10T18:03:20.894Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.884204254Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196882024-01-10T18:03:20.894Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.884290673Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196892024-01-10T18:03:20.894Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.884332941Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196902024-01-10T18:03:20.894Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.884372627Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196912024-01-10T18:03:20.895Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.884411146Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196922024-01-10T18:03:20.895Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.884451947Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196932024-01-10T18:03:20.895Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.884536461Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196942024-01-10T18:03:20.895Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.885642831Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196952024-01-10T18:03:20.895Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.885775474Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196962024-01-10T18:03:20.895Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.885999138Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196972024-01-10T18:03:20.896Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.886046899Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196982024-01-10T18:03:20.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.886677477Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
196992024-01-10T18:03:20.896Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.88671917Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197002024-01-10T18:03:20.896Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.886783442Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197012024-01-10T18:03:20.898Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.887082622Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197022024-01-10T18:03:20.907Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.887652549Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197032024-01-10T18:03:20.907Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.887709849Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197042024-01-10T18:03:20.907Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.887755058Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197052024-01-10T18:03:20.907Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.887996486Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197062024-01-10T18:03:20.907Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.888083479Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197072024-01-10T18:03:20.908Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.888165255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197082024-01-10T18:03:20.908Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.888231087Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197092024-01-10T18:03:20.908Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.888875203Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197102024-01-10T18:03:20.908Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.888964885Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197112024-01-10T18:03:20.908Z{"msg":"[0]R flush_numbers[0..12]: [341, 341, 339, 327, 327, 335, 340, 323, 340, 340, 340, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889006509Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197122024-01-10T18:03:20.908Z{"msg":"[0]R generation[0..12]: [183, 183, 182, 178, 178, 181, 182, 176, 182, 182, 182, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889037725Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197132024-01-10T18:03:20.908Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889073842Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197142024-01-10T18:03:20.908Z{"msg":"[1]R flush_numbers[0..12]: [341, 341, 339, 327, 327, 335, 340, 323, 340, 340, 340, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889121368Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197152024-01-10T18:03:20.908Z{"msg":"[1]R generation[0..12]: [183, 183, 182, 178, 178, 181, 182, 176, 182, 182, 182, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889146913Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197162024-01-10T18:03:20.908Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
197172024-01-10T18:03:20.908Z,"time":"2024-01-10T18:03:20.889188998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197182024-01-10T18:03:20.908Z{"msg":"[2]R flush_numbers[0..12]: [341, 341, 339, 327, 327, 335, 340, 323, 340, 340, 340, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889214389Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197192024-01-10T18:03:20.909Z{"msg":"[2]R generation[0..12]: [183, 183, 182, 178, 178, 181, 182, 176, 182, 182, 182, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889237431Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197202024-01-10T18:03:20.909Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889260615Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197212024-01-10T18:03:20.909Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889281853Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197222024-01-10T18:03:20.909Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889344174Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197232024-01-10T18:03:20.909Z{"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.88936911Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197242024-01-10T18:03:20.909Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889391054Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197252024-01-10T18:03:20.909Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889438712Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197262024-01-10T18:03:20.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889459768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197272024-01-10T18:03:20.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889630796Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197282024-01-10T18:03:20.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889764786Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197292024-01-10T18:03:20.909Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889842277Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197302024-01-10T18:03:20.909Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889916467Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197312024-01-10T18:03:20.910Z{"msg":"f599db80-3171-4c21-a2f3-15b402a71ab4 is now active with session: 2e1933e9-b3a9-4282-bd19-90c36a8e1f1c","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.889961162Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197322024-01-10T18:03:20.910Z2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197332024-01-10T18:03:20.910Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.890158579Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894,"session_id":"2e1933e9-b3a9-4282-bd19-90c36a8e1f1c"}
197342024-01-10T18:03:20.910Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:20.890235332Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1894}
197352024-01-10T18:03:20.911ZLoading write count information from file "/tmp/test_repair_verify.data"
197362024-01-10T18:03:20.911ZRun Repair workload
197372024-01-10T18:03:20.911Z01/30 Flush
197382024-01-10T18:03:20.911Z02/30 Write block 73 len 24576 data: 7 9 16 16 17 13
197392024-01-10T18:03:20.911Z03/30 Write block 103 len 8192 data: 9 5
197402024-01-10T18:03:20.911Z04/30 Read block 67 len 4096
197412024-01-10T18:03:20.911Z05/30 Read block 224 len 12288
197422024-01-10T18:03:20.911Z06/30 Read block 267 len 28672
197432024-01-10T18:03:20.911Z07/30 Write block 541 len 24576 data: 21 12 12 13 15 14
197442024-01-10T18:03:20.915Z08/30 Read block 303 len 36864
197452024-01-10T18:03:20.915Z09/30 Read block 533 len 32768
197462024-01-10T18:03:20.915Z10/30 Write block 57 len 4096 data: 13
197472024-01-10T18:03:20.915Z11/30 Write block 130 len 24576 data: 10 9 10 10 11 12
197482024-01-10T18:03:20.915Z12/30 Read block 392 len 24576
197492024-01-10T18:03:20.920Z13/30 Read block 320 len 40960
197502024-01-10T18:03:20.925Z14/30 Write block 50 len 4096 data: 17
197512024-01-10T18:03:20.925Z15/30 Read block 237 len 8192
197522024-01-10T18:03:20.925Z16/30 Read block 72 len 28672
197532024-01-10T18:03:20.925Z17/30 Read block 257 len 20480
197542024-01-10T18:03:20.931Z18/30 Read block 422 len 12288
197552024-01-10T18:03:20.931Z19/30 Write block 67 len 12288 data: 18 15 13
197562024-01-10T18:03:20.931Z20/30 Flush
197572024-01-10T18:03:20.936Z21/30 Write block 177 len 8192 data: 12 12
197582024-01-10T18:03:20.936Z22/30 Write block 147 len 16384 data: 16 15 14 13
197592024-01-10T18:03:20.936Z23/30 Flush
197602024-01-10T18:03:20.940Z24/30 Write block 570 len 16384 data: 13 13 13 14
197612024-01-10T18:03:20.940Z25/30 Read block 535 len 8192
197622024-01-10T18:03:20.940Z26/30 Read block 272 len 8192
197632024-01-10T18:03:20.944Z27/30 Write block 242 len 40960 data: 14 16 20 18 13 11 11 13 11 10
197642024-01-10T18:03:20.944Z28/30 Write block 591 len 16384 data: 11 10 11 14
197652024-01-10T18:03:20.944Z29/30 Write block 187 len 40960 data: 14 16 16 16 16 15 16 18 20 19
197662024-01-10T18:03:20.944Z30/30 Read block 178 len 4096
197672024-01-10T18:03:20.949Z----------------------------------------------------------------
197682024-01-10T18:03:20.949Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
197692024-01-10T18:03:20.950ZStates: Active Active Active
197702024-01-10T18:03:20.950ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
197712024-01-10T18:03:20.950Z 1 Acked 1000 Flush 0 Sent Done Done false
197722024-01-10T18:03:20.950Z 2 Acked 1001 Write 6 Sent Done Done false
197732024-01-10T18:03:20.950Z 3 Acked 1002 Write 2 Sent Done Done false
197742024-01-10T18:03:20.950Z 4 Acked 1003 Read 1 Sent Done Done false
197752024-01-10T18:03:20.950Z 5 Acked 1004 Read 3 Sent Done Done false
197762024-01-10T18:03:20.950Z 6 Acked 1005 Read 7 Sent Done Done false
197772024-01-10T18:03:20.950Z 7 Acked 1006 Write 6 Sent Done Done false
197782024-01-10T18:03:20.950Z 8 Acked 1007 Read 9 Sent Done Done false
197792024-01-10T18:03:20.950Z 9 Acked 1008 Read 8 Sent Done Done false
197802024-01-10T18:03:20.951Z 10 Acked 1009 Write 1 Sent Done Done false
197812024-01-10T18:03:20.951Z 11 Acked 1010 Write 6 Sent Done Done false
197822024-01-10T18:03:20.951Z 12 Acked 1011 Read 6 Sent Done Done false
197832024-01-10T18:03:20.951Z 13 Acked 1012 Read 10 Sent Done Done false
197842024-01-10T18:03:20.951Z 14 Acked 1013 Write 1 Sent Done Done false
197852024-01-10T18:03:20.951Z 15 Acked 1014 Read 2 Sent Done Done false
197862024-01-10T18:03:20.951Z 16 Acked 1015 Read 7 Sent Done Done false
197872024-01-10T18:03:20.951Z 17 Acked 1016 Read 5 Sent Done Done false
197882024-01-10T18:03:20.951Z 18 Acked 1017 Read 3 Sent Done Done false
197892024-01-10T18:03:20.952Z 19 Acked 1018 Write 3 Sent Done Done false
197902024-01-10T18:03:20.952Z 20 Acked 1019 Flush 0 Sent Done Done false
197912024-01-10T18:03:20.952Z 21 Acked 1020 Write 2 Sent Done Done false
197922024-01-10T18:03:20.952Z 22 Acked 1021 Write 4 Sent Done Done false
197932024-01-10T18:03:20.952Z 23 Acked 1022 Flush 0 Sent Done Done false
197942024-01-10T18:03:20.953Z 24 Acked 1023 Write 4 Sent Done Done false
197952024-01-10T18:03:20.953Z 25 Acked 1024 Read 2 Sent Done Done false
197962024-01-10T18:03:20.953Z 26 Acked 1025 Read 2 Sent Done Done false
197972024-01-10T18:03:20.953Z 27 Acked 1026 Write 10 Sent Done Done false
197982024-01-10T18:03:20.953Z 28 Acked 1027 Write 4 Sent Done Done false
197992024-01-10T18:03:20.962Z 29 Acked 1028 Write 10 Sent Done Done false
198002024-01-10T18:03:20.963Z 30 Acked 1029 Read 1 Sent Done Sent false
198012024-01-10T18:03:20.963Z STATES DS:0 DS:1 DS:2 TOTAL
198022024-01-10T18:03:20.963Z New 0 0 0 0
198032024-01-10T18:03:20.963Z Sent 30 0 1 31
198042024-01-10T18:03:20.963Z Done 0 30 29 59
198052024-01-10T18:03:20.963Z Skipped 0 0 0 0
198062024-01-10T18:03:20.963Z Error 0 0 0 0
198072024-01-10T18:03:20.963ZLast Flush: 0 1022 1022
198082024-01-10T18:03:20.964ZDownstairs last five completed:
198092024-01-10T18:03:20.964ZUpstairs last five completed: 30 29 28 27 26
198102024-01-10T18:03:20.964ZWrote out file "/tmp/test_repair_verify.data"
198112024-01-10T18:03:21.383Z
198122024-01-10T18:03:21.384ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198132024-01-10T18:03:21.393ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198142024-01-10T18:03:21.393Z 0 000-019 183 183 183 341 341 341 F F F
198152024-01-10T18:03:21.393Z 1 020-039 183 183 183 341 341 341 F F F
198162024-01-10T18:03:21.393Z 2 040-059 182 184 184 339 343 343 F F F <---
198172024-01-10T18:03:21.393Z 3 060-079 178 184 184 327 343 343 F F F <---
198182024-01-10T18:03:21.395Z 4 080-099 178 178 178 327 327 327 F F F
198192024-01-10T18:03:21.395Z 5 100-119 181 184 184 335 343 343 F F F <---
198202024-01-10T18:03:21.395Z 6 120-139 182 184 184 340 343 343 F F F <---
198212024-01-10T18:03:21.395Z 7 140-159 176 184 184 323 344 344 F F F <---
198222024-01-10T18:03:21.395Z 8 160-179 182 184 184 340 344 344 F F F <---
198232024-01-10T18:03:21.396Z 9 180-199 182 182 182 340 340 340 F T T <---
198242024-01-10T18:03:21.396Z 10 200-219 182 182 182 340 340 340 F F F
198252024-01-10T18:03:21.396Z 11 220-239 174 174 174 319 319 319 F F F
198262024-01-10T18:03:21.396Z 12 240-259 182 182 182 337 337 337 F T T <---
198272024-01-10T18:03:21.396Z 13 260-279 182 182 182 337 337 337 F F F
198282024-01-10T18:03:21.396Z 14 280-299 182 182 182 338 338 338 F F F
198292024-01-10T18:03:21.396Z 15 300-319 182 182 182 338 338 338 F F F
198302024-01-10T18:03:21.396Z 16 320-339 183 183 183 341 341 341 F F F
198312024-01-10T18:03:21.396Z 17 340-359 181 181 181 335 335 335 F F F
198322024-01-10T18:03:21.396Z 18 360-379 181 181 181 335 335 335 F F F
198332024-01-10T18:03:21.396Z 19 380-399 182 182 182 340 340 340 F F F
198342024-01-10T18:03:21.396Z 20 400-419 174 174 174 319 319 319 F F F
198352024-01-10T18:03:21.396Z 21 420-439 181 181 181 335 335 335 F F F
198362024-01-10T18:03:21.396Z 22 440-459 180 180 180 331 331 331 F F F
198372024-01-10T18:03:21.396Z 23 460-479 180 180 180 331 331 331 F F F
198382024-01-10T18:03:21.397Z 24 480-499 182 182 182 338 338 338 F F F
198392024-01-10T18:03:21.397Z 25 500-519 182 182 182 340 340 340 F F F
198402024-01-10T18:03:21.397Z 26 520-539 181 181 181 335 335 335 F F F
198412024-01-10T18:03:21.397Z 27 540-559 183 184 184 341 343 343 F F F <---
198422024-01-10T18:03:21.397Z 28 560-579 180 180 180 332 332 332 F T T <---
198432024-01-10T18:03:21.397Z 29 580-599 177 177 177 324 324 324 F T T <---
198442024-01-10T18:03:21.397ZMax gen: 184, Max flush: 344
198452024-01-10T18:03:21.397ZError: Difference in extent metadata found!
198462024-01-10T18:03:21.398ZA Difference in extent metadata is expected here
198472024-01-10T18:03:21.398ZOn loop 92
198482024-01-10T18:03:21.398Z
198492024-01-10T18:03:21.398ZVerifying data now
198502024-01-10T18:03:26.112ZNo change after verify
198512024-01-10T18:03:26.112ZLoop: 92 Downstairs dump after verify (and repair):
198522024-01-10T18:03:26.134ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198532024-01-10T18:03:26.134Z 0 000-019 183 183 183 341 341 341 F F F
198542024-01-10T18:03:26.134Z 1 020-039 183 183 183 341 341 341 F F F
198552024-01-10T18:03:26.134Z 2 040-059 184 184 184 343 343 343 F F F
198562024-01-10T18:03:26.134Z 3 060-079 184 184 184 343 343 343 F F F
198572024-01-10T18:03:26.135Z 4 080-099 178 178 178 327 327 327 F F F
198582024-01-10T18:03:26.135Z 5 100-119 184 184 184 343 343 343 F F F
198592024-01-10T18:03:26.138Z 6 120-139 184 184 184 343 343 343 F F F
198602024-01-10T18:03:26.138Z 7 140-159 184 184 184 344 344 344 F F F
198612024-01-10T18:03:26.138Z 8 160-179 184 184 184 344 344 344 F F F
198622024-01-10T18:03:26.152Z 9 180-199 185 185 185 345 345 345 F F F
198632024-01-10T18:03:26.152Z 10 200-219 182 182 182 340 340 340 F F F
198642024-01-10T18:03:26.152Z 11 220-239 174 174 174 319 319 319 F F F
198652024-01-10T18:03:26.153Z 12 240-259 185 185 185 345 345 345 F F F
198662024-01-10T18:03:26.153Z 13 260-279 182 182 182 337 337 337 F F F
198672024-01-10T18:03:26.153Z 14 280-299 182 182 182 338 338 338 F F F
198682024-01-10T18:03:26.153Z 15 300-319 182 182 182 338 338 338 F F F
198692024-01-10T18:03:26.153Z 16 320-339 183 183 183 341 341 341 F F F
198702024-01-10T18:03:26.153Z 17 340-359 181 181 181 335 335 335 F F F
198712024-01-10T18:03:26.153Z 18 360-379 181 181 181 335 335 335 F F F
198722024-01-10T18:03:26.154Z 19 380-399 182 182 182 340 340 340 F F F
198732024-01-10T18:03:26.154Z 20 400-419 174 174 174 319 319 319 F F F
198742024-01-10T18:03:26.154Z 21 420-439 181 181 181 335 335 335 F F F
198752024-01-10T18:03:26.154Z 22 440-459 180 180 180 331 331 331 F F F
198762024-01-10T18:03:26.154Z 23 460-479 180 180 180 331 331 331 F F F
198772024-01-10T18:03:26.154Z 24 480-499 182 182 182 338 338 338 F F F
198782024-01-10T18:03:26.154Z 25 500-519 182 182 182 340 340 340 F F F
198792024-01-10T18:03:26.154Z 26 520-539 181 181 181 335 335 335 F F F
198802024-01-10T18:03:26.154Z 27 540-559 184 184 184 343 343 343 F F F
198812024-01-10T18:03:26.155Z 28 560-579 185 185 185 345 345 345 F F F
198822024-01-10T18:03:26.155Z 29 580-599 185 185 185 345 345 345 F F F
198832024-01-10T18:03:26.155ZMax gen: 185, Max flush: 345
198842024-01-10T18:03:26.158Z
198852024-01-10T18:03:26.159ZBegin loop 93
198862024-01-10T18:03:26.159ZDownstairs to restart: 2
198872024-01-10T18:03:26.171Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.168758519Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902}
198882024-01-10T18:03:26.177Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.169350638Z","hostname":"Crucible runtime is spawned
198892024-01-10T18:03:26.177Zip-10-150-1-7.us-west-2.compute.internal","pid":1902}
198902024-01-10T18:03:26.177Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.169535092Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902}
198912024-01-10T18:03:26.177Z{"msg":"Crucible 2e3e5771-a358-4161-88e0-04efff88f5e0 has session id: 486edc3b-0d69-4ca5-a679-d74eb16194a2","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.169612979Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
198922024-01-10T18:03:26.178Z{"msg":"Upstairs opts: Upstairs UUID: 2e3e5771-a358-4161-88e0-04efff88f5e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.169705465Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
198932024-01-10T18:03:26.178Z{"msg":"Crucible stats registered with UUID: 2e3e5771-a358-4161-88e0-04efff88f5e0","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.169777349Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
198942024-01-10T18:03:26.178Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.16981998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902}
198952024-01-10T18:03:26.178Z{"msg":"2e3e5771-a358-4161-88e0-04efff88f5e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.169913886Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
198962024-01-10T18:03:26.178Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.169955654Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
198972024-01-10T18:03:26.179Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170001413Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
198982024-01-10T18:03:26.179Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170064857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
198992024-01-10T18:03:26.179Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170092808Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199002024-01-10T18:03:26.179Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170114418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199012024-01-10T18:03:26.179Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170140334Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199022024-01-10T18:03:26.179Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170230175Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199032024-01-10T18:03:26.179Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170312522Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199042024-01-10T18:03:26.179Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170362804Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199052024-01-10T18:03:26.179Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170478059Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199062024-01-10T18:03:26.179Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170585199Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199072024-01-10T18:03:26.179Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.170636923Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199082024-01-10T18:03:26.179Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.171857357Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199092024-01-10T18:03:26.179Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.171924741Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199102024-01-10T18:03:26.179Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.17201792Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199112024-01-10T18:03:26.179Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.172089535Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199122024-01-10T18:03:26.179Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.172346783Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199132024-01-10T18:03:26.185Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.172419889Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199142024-01-10T18:03:26.185Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.172876517Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199152024-01-10T18:03:26.185Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.173059457Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199162024-01-10T18:03:26.185Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.173598416Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199172024-01-10T18:03:26.185Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.173665871Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199182024-01-10T18:03:26.185Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.17369345Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199192024-01-10T18:03:26.185Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.173716615Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199202024-01-10T18:03:26.185Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.173737619Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199212024-01-10T18:03:26.185Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.173780312Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199222024-01-10T18:03:26.187Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.173807513Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199232024-01-10T18:03:26.187Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.17464147Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199242024-01-10T18:03:26.187Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.174762693Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199252024-01-10T18:03:26.187Z{"msg":"[0]R flush_numbers[0..12]: [341, 341, 343, 343, 327, 343, 343, 344, 344, 345, 340, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.17479026Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199262024-01-10T18:03:26.187Z{"msg":"[0]R generation[0..12]: [183, 183, 184, 184, 178, 184, 184, 184, 184, 185, 182, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.174814235Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199272024-01-10T18:03:26.187Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.174840279Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199282024-01-10T18:03:26.187Z{"msg":"[1]R flush_numbers[0..12]: [341, 341, 343, 343, 327, 343, 343, 344, 344, 345, 340, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.174863815Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199292024-01-10T18:03:26.187Z{"msg":"[1]R generation[0..12]: [183, 183, 184, 184, 178, 184, 184, 184, 184, 185, 182, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.174886752Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199302024-01-10T18:03:26.187ZWait for a query_work_queue command to finish before sending IO
199312024-01-10T18:03:26.187Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.174937072Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199322024-01-10T18:03:26.188Z{"msg":"[2]R flush_numbers[0..12]: [341, 341, 343, 343, 327, 343, 343, 344, 344, 345, 340, 319]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.174962538Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199332024-01-10T18:03:26.188Z{"msg":"[2]R generation[0..12]: [183, 183, 184, 184, 178, 184, 184, 184, 184, 185, 182, 174]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175052078Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199342024-01-10T18:03:26.188Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175081173Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199352024-01-10T18:03:26.188Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175123379Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199362024-01-10T18:03:26.188Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175147377Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199372024-01-10T18:03:26.188Z{"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175187302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199382024-01-10T18:03:26.189Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175234462Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199392024-01-10T18:03:26.189Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175258087Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199402024-01-10T18:03:26.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175278087Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199412024-01-10T18:03:26.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175318136Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199422024-01-10T18:03:26.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175341219Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199432024-01-10T18:03:26.189Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175363187Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199442024-01-10T18:03:26.189Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175384452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199452024-01-10T18:03:26.189Z{"msg":"2e3e5771-a358-4161-88e0-04efff88f5e0 is now active with session: 486edc3b-0d69-4ca5-a679-d74eb16194a2","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175424569Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199462024-01-10T18:03:26.189Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175445701Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902,"session_id":"486edc3b-0d69-4ca5-a679-d74eb16194a2"}
199472024-01-10T18:03:26.190Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:26.175465933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1902}
199482024-01-10T18:03:26.190ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
199492024-01-10T18:03:26.190ZLoading write count information from file "/tmp/test_repair_verify.data"
199502024-01-10T18:03:26.190ZRun Repair workload
199512024-01-10T18:03:26.190Z01/30 Write block 219 len 36864 data: 11 11 13 12 10 8 11 11 11
199522024-01-10T18:03:26.190Z02/30 Read block 210 len 24576
199532024-01-10T18:03:26.193Z03/30 Write block 97 len 4096 data: 18
199542024-01-10T18:03:26.193Z04/30 Read block 128 len 20480
199552024-01-10T18:03:26.193Z05/30 Read block 487 len 16384
199562024-01-10T18:03:26.193Z06/30 Read block 406 len 8192
199572024-01-10T18:03:26.193Z07/30 Write block 458 len 16384 data: 13 13 13 12
199582024-01-10T18:03:26.193Z08/30 Write block 42 len 20480 data: 14 12 14 14 12
199592024-01-10T18:03:26.193Z09/30 Read block 300 len 4096
199602024-01-10T18:03:26.197Z10/30 Read block 85 len 4096
199612024-01-10T18:03:26.197Z11/30 Read block 572 len 12288
199622024-01-10T18:03:26.197Z12/30 Write block 93 len 4096 data: 13
199632024-01-10T18:03:26.201Z13/30 Read block 370 len 20480
199642024-01-10T18:03:26.201Z14/30 Write block 558 len 24576 data: 12 11 12 14 15 16
199652024-01-10T18:03:26.201Z15/30 Read block 217 len 36864
199662024-01-10T18:03:26.204Z16/30 Read block 432 len 36864
199672024-01-10T18:03:26.207Z17/30 Read block 1 len 20480
199682024-01-10T18:03:26.210Z18/30 Read block 185 len 8192
199692024-01-10T18:03:26.213Z19/30 Read block 415 len 4096
199702024-01-10T18:03:26.213Z20/30 Write block 37 len 20480 data: 14 12 11 10 13
199712024-01-10T18:03:26.213Z21/30 Write block 11 len 12288 data: 13 10 10
199722024-01-10T18:03:26.216Z22/30 Write block 335 len 4096 data: 12
199732024-01-10T18:03:26.216Z23/30 Write block 471 len 36864 data: 9 8 9 10 8 7 7 5 8
199742024-01-10T18:03:26.216Z24/30 Read block 569 len 8192
199752024-01-10T18:03:26.293Z25/30 Flush
199762024-01-10T18:03:26.298Z26/30 Write block 458 len 36864 data: 14 14 14 13 11 14 14 12 16
199772024-01-10T18:03:26.298Z27/30 Write block 407 len 28672 data: 21 17 15 15 16 13 15
199782024-01-10T18:03:26.298Z28/30 Read block 436 len 12288
199792024-01-10T18:03:26.298Z29/30 Read block 492 len 12288
199802024-01-10T18:03:26.298Z30/30 Write block 243 len 40960 data: 17 21 19 14 12 12 14 12 11 11
199812024-01-10T18:03:26.298Z----------------------------------------------------------------
199822024-01-10T18:03:26.298Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
199832024-01-10T18:03:26.300ZStates: Active Active Active
199842024-01-10T18:03:26.300ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
199852024-01-10T18:03:26.300Z 1 Acked 1000 Write 9 Done Done Done false
199862024-01-10T18:03:26.300Z 2 Acked 1001 Read 6 Done Done Done false
199872024-01-10T18:03:26.300Z 3 Acked 1002 Write 1 Done Done Done false
199882024-01-10T18:03:26.300Z 4 Acked 1003 Read 5 Done Done Done false
199892024-01-10T18:03:26.300Z 5 Acked 1004 Read 4 Done Done Sent false
199902024-01-10T18:03:26.300Z 6 Acked 1005 Read 2 Done Done Sent false
199912024-01-10T18:03:26.300Z 7 Acked 1006 Write 4 Done Done Sent false
199922024-01-10T18:03:26.300Z 8 Acked 1007 Write 5 Done Done Sent false
199932024-01-10T18:03:26.300Z 9 Acked 1008 Read 1 Done Done Sent false
199942024-01-10T18:03:26.300Z 10 Acked 1009 Read 1 Done Done Sent false
199952024-01-10T18:03:26.300Z 11 Acked 1010 Read 3 Done Done Sent false
199962024-01-10T18:03:26.300Z 12 Acked 1011 Write 1 Done Done Sent false
199972024-01-10T18:03:26.300Z 13 Acked 1012 Read 5 Done Done Sent false
199982024-01-10T18:03:26.301Z 14 Acked 1013 Write 6 Done Done Sent false
199992024-01-10T18:03:26.301Z 15 Acked 1014 Read 9 Done Done Sent false
200002024-01-10T18:03:26.301Z 16 Acked 1015 Read 9 Done Done Sent false
200012024-01-10T18:03:26.301Z 17 Acked 1016 Read 5 Done Done Sent false
200022024-01-10T18:03:26.301Z 18 Acked 1017 Read 2 Done Done Sent false
200032024-01-10T18:03:26.301Z 19 Acked 1018 Read 1 Done Done Sent false
200042024-01-10T18:03:26.301Z 20 Acked 1019 Write 5 Done Done Sent false
200052024-01-10T18:03:26.301Z 21 Acked 1020 Write 3 Done Done Sent false
200062024-01-10T18:03:26.301Z 22 Acked 1021 Write 1 Done Done Sent false
200072024-01-10T18:03:26.302Z 23 Acked 1022 Write 9 Done Done Sent false
200082024-01-10T18:03:26.302Z 24 Acked 1023 Read 2 Done Done Sent false
200092024-01-10T18:03:26.302Z 25 Acked 1024 Flush 0 Done Done Sent false
200102024-01-10T18:03:26.302Z 26 Acked 1025 Write 9 Done Done Sent false
200112024-01-10T18:03:26.302Z 27 Acked 1026 Write 7 Done Done Sent false
200122024-01-10T18:03:26.302Z 28 Acked 1027 Read 3 Done Done Sent false
200132024-01-10T18:03:26.302Z 29 Acked 1028 Read 3 Done Done Sent false
200142024-01-10T18:03:26.302Z 30 Acked 1029 Write 10 Sent Sent Sent false
200152024-01-10T18:03:26.302Z STATES DS:0 DS:1 DS:2 TOTAL
200162024-01-10T18:03:26.302Z New 0 0 0 0
200172024-01-10T18:03:26.302Z Sent 1 1 26 28
200182024-01-10T18:03:26.302Z Done 29 29 4 62
200192024-01-10T18:03:26.302Z Skipped 0 0 0 0
200202024-01-10T18:03:26.302Z Error 0 0 0 0
200212024-01-10T18:03:26.302ZLast Flush: 1024 1024 0
200222024-01-10T18:03:26.302ZDownstairs last five completed:
200232024-01-10T18:03:26.302ZUpstairs last five completed: 30 29 28 27 26
200242024-01-10T18:03:26.302ZWrote out file "/tmp/test_repair_verify.data"
200252024-01-10T18:03:26.302Z
200262024-01-10T18:03:26.302ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
200272024-01-10T18:03:26.302ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200282024-01-10T18:03:26.302Z 0 000-019 186 186 183 346 346 341 F F F <---
200292024-01-10T18:03:26.302Z 1 020-039 186 186 183 346 346 341 F F F <---
200302024-01-10T18:03:26.303Z 2 040-059 186 186 184 346 346 343 F F F <---
200312024-01-10T18:03:26.303Z 3 060-079 184 184 184 343 343 343 F F F
200322024-01-10T18:03:26.304Z 4 080-099 186 186 178 346 346 327 F F T <---
200332024-01-10T18:03:26.304Z 5 100-119 184 184 184 343 343 343 F F F
200342024-01-10T18:03:26.304Z 6 120-139 184 184 184 343 343 343 F F F
200352024-01-10T18:03:26.304Z 7 140-159 184 184 184 344 344 344 F F F
200362024-01-10T18:03:26.304Z 8 160-179 184 184 184 344 344 344 F F F
200372024-01-10T18:03:26.304Z 9 180-199 185 185 185 345 345 345 F F F
200382024-01-10T18:03:26.304Z 10 200-219 186 186 182 346 346 340 F F T <---
200392024-01-10T18:03:26.304Z 11 220-239 186 186 174 346 346 319 F F T <---
200402024-01-10T18:03:26.304Z 12 240-259 185 185 185 345 345 345 T T F <---
200412024-01-10T18:03:26.304Z 13 260-279 182 182 182 337 337 337 F F F
200422024-01-10T18:03:26.304Z 14 280-299 182 182 182 338 338 338 F F F
200432024-01-10T18:03:26.304Z 15 300-319 182 182 182 338 338 338 F F F
200442024-01-10T18:03:26.304Z 16 320-339 186 186 183 346 346 341 F F F <---
200452024-01-10T18:03:26.304Z 17 340-359 181 181 181 335 335 335 F F F
200462024-01-10T18:03:26.304Z 18 360-379 181 181 181 335 335 335 F F F
200472024-01-10T18:03:26.304Z 19 380-399 182 182 182 340 340 340 F F F
200482024-01-10T18:03:26.304Z 20 400-419 174 174 174 319 319 319 T T F <---
200492024-01-10T18:03:26.304Z 21 420-439 181 181 181 335 335 335 F F F
200502024-01-10T18:03:26.304Z 22 440-459 186 186 180 346 346 331 T T F <---
200512024-01-10T18:03:26.304Z 23 460-479 186 186 180 346 346 331 T T F <---
200522024-01-10T18:03:26.304Z 24 480-499 182 182 182 338 338 338 F F F
200532024-01-10T18:03:26.304Z 25 500-519 182 182 182 340 340 340 F F F
200542024-01-10T18:03:26.306Z 26 520-539 181 181 181 335 335 335 F F F
200552024-01-10T18:03:26.306Z 27 540-559 186 186 184 346 346 343 F F F <---
200562024-01-10T18:03:26.306Z 28 560-579 186 186 185 346 346 345 F F F <---
200572024-01-10T18:03:26.306Z 29 580-599 185 185 185 345 345 345 F F F
200582024-01-10T18:03:26.306ZMax gen: 186, Max flush: 346
200592024-01-10T18:03:26.306ZError: Difference in extent metadata found!
200602024-01-10T18:03:26.306ZA Difference in extent metadata is expected here
200612024-01-10T18:03:26.306ZOn loop 93
200622024-01-10T18:03:26.306Z
200632024-01-10T18:03:26.306ZVerifying data now
200642024-01-10T18:03:32.543ZNo change after verify
200652024-01-10T18:03:32.543ZLoop: 93 Downstairs dump after verify (and repair):
200662024-01-10T18:03:32.566ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200672024-01-10T18:03:32.566Z 0 000-019 186 186 186 346 346 346 F F F
200682024-01-10T18:03:32.566Z 1 020-039 186 186 186 346 346 346 F F F
200692024-01-10T18:03:32.566Z 2 040-059 186 186 186 346 346 346 F F F
200702024-01-10T18:03:32.566Z 3 060-079 184 184 184 343 343 343 F F F
200712024-01-10T18:03:32.566Z 4 080-099 186 186 186 346 346 346 F F F
200722024-01-10T18:03:32.566Z 5 100-119 184 184 184 343 343 343 F F F
200732024-01-10T18:03:32.570Z 6 120-139 184 184 184 343 343 343 F F F
200742024-01-10T18:03:32.570Z 7 140-159 184 184 184 344 344 344 F F F
200752024-01-10T18:03:32.570Z 8 160-179 184 184 184 344 344 344 F F F
200762024-01-10T18:03:32.570Z 9 180-199 185 185 185 345 345 345 F F F
200772024-01-10T18:03:32.570Z 10 200-219 186 186 186 346 346 346 F F F
200782024-01-10T18:03:32.570Z 11 220-239 186 186 186 346 346 346 F F F
200792024-01-10T18:03:32.570Z 12 240-259 187 187 187 347 347 347 F F F
200802024-01-10T18:03:32.570Z 13 260-279 182 182 182 337 337 337 F F F
200812024-01-10T18:03:32.570Z 14 280-299 182 182 182 338 338 338 F F F
200822024-01-10T18:03:32.570Z 15 300-319 182 182 182 338 338 338 F F F
200832024-01-10T18:03:32.570Z 16 320-339 186 186 186 346 346 346 F F F
200842024-01-10T18:03:32.570Z 17 340-359 181 181 181 335 335 335 F F F
200852024-01-10T18:03:32.570Z 18 360-379 181 181 181 335 335 335 F F F
200862024-01-10T18:03:32.570Z 19 380-399 182 182 182 340 340 340 F F F
200872024-01-10T18:03:32.571Z 20 400-419 187 187 187 347 347 347 F F F
200882024-01-10T18:03:32.571Z 21 420-439 181 181 181 335 335 335 F F F
200892024-01-10T18:03:32.571Z 22 440-459 187 187 187 347 347 347 F F F
200902024-01-10T18:03:32.571Z 23 460-479 187 187 187 347 347 347 F F F
200912024-01-10T18:03:32.571Z 24 480-499 182 182 182 338 338 338 F F F
200922024-01-10T18:03:32.571Z 25 500-519 182 182 182 340 340 340 F F F
200932024-01-10T18:03:32.571Z 26 520-539 181 181 181 335 335 335 F F F
200942024-01-10T18:03:32.572Z 27 540-559 186 186 186 346 346 346 F F F
200952024-01-10T18:03:32.572Z 28 560-579 186 186 186 346 346 346 F F F
200962024-01-10T18:03:32.572Z 29 580-599 185 185 185 345 345 345 F F F
200972024-01-10T18:03:32.572ZMax gen: 187, Max flush: 347
200982024-01-10T18:03:32.576Z
200992024-01-10T18:03:32.576ZBegin loop 94
201002024-01-10T18:03:32.576ZDownstairs to restart: 1
201012024-01-10T18:03:32.601Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.597944523Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910}
201022024-01-10T18:03:32.601Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.59844617Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910}
201032024-01-10T18:03:32.613Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.598593876Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910}
201042024-01-10T18:03:32.613Z{"msg":"Crucible 06df8da1-b073-4c81-a167-2c979bea322a has session id: d1c48c81-f8c3-45c5-9af4-169b1c9444bb","v":0,"name":"crucible","level":30Crucible runtime is spawned
201052024-01-10T18:03:32.613Z,"time":"2024-01-10T18:03:32.598676312Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201062024-01-10T18:03:32.618Z{"msg":"Upstairs opts: Upstairs UUID: 06df8da1-b073-4c81-a167-2c979bea322a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.598820885Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201072024-01-10T18:03:32.618Z{"msg":"Crucible stats registered with UUID: 06df8da1-b073-4c81-a167-2c979bea322a","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.59888089Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201082024-01-10T18:03:32.618Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.59892169Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910}
201092024-01-10T18:03:32.618Z{"msg":"06df8da1-b073-4c81-a167-2c979bea322a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.598971999Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201102024-01-10T18:03:32.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.599010774Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201112024-01-10T18:03:32.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.599220747Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201122024-01-10T18:03:32.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.599288292Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201132024-01-10T18:03:32.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.599356927Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201142024-01-10T18:03:32.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.599469714Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201152024-01-10T18:03:32.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.599532637Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201162024-01-10T18:03:32.619Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.5995805Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201172024-01-10T18:03:32.619Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.599629149Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201182024-01-10T18:03:32.619Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.599675116Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201192024-01-10T18:03:32.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.599874771Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201202024-01-10T18:03:32.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.599958937Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201212024-01-10T18:03:32.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.600070223Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201222024-01-10T18:03:32.619Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.60073708Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201232024-01-10T18:03:32.619Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.60084879Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201242024-01-10T18:03:32.619Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.600907385Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201252024-01-10T18:03:32.619Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.601032178Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201262024-01-10T18:03:32.625Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.601487631Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201272024-01-10T18:03:32.625Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.60162832Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201282024-01-10T18:03:32.626Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.601829113Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201292024-01-10T18:03:32.626Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.601975114Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201302024-01-10T18:03:32.626Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.602623594Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201312024-01-10T18:03:32.626Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.602908622Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201322024-01-10T18:03:32.626Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.602970482Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201332024-01-10T18:03:32.626Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603037217Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201342024-01-10T18:03:32.626Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603077644Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201352024-01-10T18:03:32.626Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603132795Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201362024-01-10T18:03:32.629Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603176129Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201372024-01-10T18:03:32.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603467051Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201382024-01-10T18:03:32.629Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603589995Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201392024-01-10T18:03:32.629Z{"msg":"[0]R flush_numbers[0..12]: [346, 346, 346, 343, 346, 343, 343, 344, 344, 345, 346, 346]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603624787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201402024-01-10T18:03:32.629Z{"msg":"[0]R generation[0..12]: [186, 186, 186, 184, 186, 184, 184, 184, 184, 185, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603684052Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201412024-01-10T18:03:32.629Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603719261Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201422024-01-10T18:03:32.630Z{"msg":"[1]R flush_numbers[0..12]: [346, 346, 346, 343, 346, 343, 343, 344, 344, 345, 346, 346]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
201432024-01-10T18:03:32.630Z,"level":30,"time":"2024-01-10T18:03:32.603780489Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201442024-01-10T18:03:32.630Z{"msg":"[1]R generation[0..12]: [186, 186, 186, 184, 186, 184, 184, 184, 184, 185, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603819467Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201452024-01-10T18:03:32.630Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603875302Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201462024-01-10T18:03:32.630Z{"msg":"[2]R flush_numbers[0..12]: [346, 346, 346, 343, 346, 343, 343, 344, 344, 345, 346, 346]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.60390934Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201472024-01-10T18:03:32.630Z{"msg":"[2]R generation[0..12]: [186, 186, 186, 184, 186, 184, 184, 184, 184, 185, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.603966219Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201482024-01-10T18:03:32.630Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604078201Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201492024-01-10T18:03:32.630Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604108006Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201502024-01-10T18:03:32.630Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604170039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201512024-01-10T18:03:32.630Z{"msg":"Next flush: 348","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604201077Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201522024-01-10T18:03:32.630Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604257557Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201532024-01-10T18:03:32.630Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.60428369Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201542024-01-10T18:03:32.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604343248Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201552024-01-10T18:03:32.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604374862Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201562024-01-10T18:03:32.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
201572024-01-10T18:03:32.630Z,"time":"2024-01-10T18:03:32.604459525Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201582024-01-10T18:03:32.630Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604576797Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201592024-01-10T18:03:32.630Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604636764Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201602024-01-10T18:03:32.630Z{"msg":"06df8da1-b073-4c81-a167-2c979bea322a is now active with session: d1c48c81-f8c3-45c5-9af4-169b1c9444bb","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604669929Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201612024-01-10T18:03:32.630Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604722402Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910,"session_id":"d1c48c81-f8c3-45c5-9af4-169b1c9444bb"}
201622024-01-10T18:03:32.630Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:32.604750363Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1910}
201632024-01-10T18:03:32.630ZLoading write count information from file "/tmp/test_repair_verify.data"
201642024-01-10T18:03:32.630ZRun Repair workload
201652024-01-10T18:03:32.640Z01/30 Write block 582 len 24576 data: 15 15 12 10 13 13
201662024-01-10T18:03:32.640Z02/30 Read block 525 len 28672
201672024-01-10T18:03:32.640Z03/30 Write block 400 len 36864 data: 11 13 12 16 17 20 21 22 18
201682024-01-10T18:03:32.640Z04/30 Read block 182 len 4096
201692024-01-10T18:03:32.640Z05/30 Read block 334 len 24576
201702024-01-10T18:03:32.640Z06/30 Write block 335 len 28672 data: 13 12 12 12 15 12 13
201712024-01-10T18:03:32.640Z07/30 Write block 253 len 20480 data: 10 8 8 11 14
201722024-01-10T18:03:32.641Z08/30 Write block 106 len 12288 data: 9 8 8
201732024-01-10T18:03:32.641Z09/30 Flush
201742024-01-10T18:03:32.641Z10/30 Write block 244 len 20480 data: 22 20 15 13 13
201752024-01-10T18:03:32.641Z11/30 Read block 145 len 20480
201762024-01-10T18:03:32.641Z12/30 Read block 96 len 24576
201772024-01-10T18:03:32.641Z13/30 Write block 526 len 20480 data: 14 13 13 14 15
201782024-01-10T18:03:32.642Z14/30 Flush
201792024-01-10T18:03:32.642Z15/30 Write block 438 len 16384 data: 15 15 13 15
201802024-01-10T18:03:32.642Z16/30 Write block 310 len 8192 data: 7 9
201812024-01-10T18:03:32.642Z17/30 Flush
201822024-01-10T18:03:32.642Z18/30 Write block 298 len 20480 data: 23 19 16 15 15
201832024-01-10T18:03:32.645Z19/30 Read block 383 len 8192
201842024-01-10T18:03:32.645Z20/30 Write block 239 len 28672 data: 11 11 13 15 18 23 21
201852024-01-10T18:03:32.645Z21/30 Write block 81 len 40960 data: 15 14 16 12 12 10 10 8 9 10
201862024-01-10T18:03:32.649Z22/30 Write block 472 len 4096 data: 9
201872024-01-10T18:03:32.649Z23/30 Read block 448 len 12288
201882024-01-10T18:03:32.649Z24/30 Write block 227 len 40960 data: 12 11 11 10 7 6 7 9 10 10
201892024-01-10T18:03:32.656Z25/30 Read block 528 len 16384
201902024-01-10T18:03:32.656Z26/30 Read block 402 len 28672
201912024-01-10T18:03:32.656Z27/30 Read block 276 len 8192
201922024-01-10T18:03:32.659Z28/30 Write block 505 len 36864 data: 11 13 13 9 17 16 19 19 17
201932024-01-10T18:03:32.659Z29/30 Write block 371 len 28672 data: 11 12 12 11 12 10 10
201942024-01-10T18:03:32.662Z30/30 Write block 256 len 8192 data: 12 15
201952024-01-10T18:03:32.662Z----------------------------------------------------------------
201962024-01-10T18:03:32.662Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
201972024-01-10T18:03:32.663ZStates: Active Active Active
201982024-01-10T18:03:32.663ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
201992024-01-10T18:03:32.663Z 1 Acked 1000 Write 6 Done Done Done false
202002024-01-10T18:03:32.663Z 2 Acked 1001 Read 7 Done Done Done false
202012024-01-10T18:03:32.663Z 3 Acked 1002 Write 9 Done Done Done false
202022024-01-10T18:03:32.663Z 4 Acked 1003 Read 1 Done Done Done false
202032024-01-10T18:03:32.663Z 5 Acked 1004 Read 6 Done Sent Done false
202042024-01-10T18:03:32.663Z 6 Acked 1005 Write 7 Done Sent Done false
202052024-01-10T18:03:32.663Z 7 Acked 1006 Write 5 Done Sent Done false
202062024-01-10T18:03:32.663Z 8 Acked 1007 Write 3 Done Sent Done false
202072024-01-10T18:03:32.663Z 9 Acked 1008 Flush 0 Done Sent Done false
202082024-01-10T18:03:32.664Z 10 Acked 1009 Write 5 Done Sent Done false
202092024-01-10T18:03:32.664Z 11 Acked 1010 Read 5 Done Sent Done false
202102024-01-10T18:03:32.664Z 12 Acked 1011 Read 6 Done Sent Done false
202112024-01-10T18:03:32.664Z 13 Acked 1012 Write 5 Done Sent Done false
202122024-01-10T18:03:32.664Z 14 Acked 1013 Flush 0 Done Sent Done false
202132024-01-10T18:03:32.664Z 15 Acked 1014 Write 4 Done Sent Done false
202142024-01-10T18:03:32.664Z 16 Acked 1015 Write 2 Done Sent Done false
202152024-01-10T18:03:32.664Z 17 Acked 1016 Flush 0 Done Sent Done false
202162024-01-10T18:03:32.664Z 18 Acked 1017 Write 5 Done Sent Done false
202172024-01-10T18:03:32.664Z 19 Acked 1018 Read 2 Done Sent Done false
202182024-01-10T18:03:32.664Z 20 Acked 1019 Write 7 Done Sent Done false
202192024-01-10T18:03:32.664Z 21 Acked 1020 Write 10 Done Sent Done false
202202024-01-10T18:03:32.664Z 22 Acked 1021 Write 1 Done Sent Done false
202212024-01-10T18:03:32.664Z 23 Acked 1022 Read 3 Done Sent Done false
202222024-01-10T18:03:32.664Z 24 Acked 1023 Write 10 Done Sent Done false
202232024-01-10T18:03:32.664Z 25 Acked 1024 Read 4 Done Sent Done false
202242024-01-10T18:03:32.664Z 26 Acked 1025 Read 7 Done Sent Done false
202252024-01-10T18:03:32.664Z 27 Acked 1026 Read 2 Done Sent Done false
202262024-01-10T18:03:32.664Z 28 Acked 1027 Write 9 Done Sent Done false
202272024-01-10T18:03:32.664Z 29 Acked 1028 Write 7 Sent Sent Sent false
202282024-01-10T18:03:32.664Z 30 Acked 1029 Write 2 Sent Sent Sent false
202292024-01-10T18:03:32.664Z STATES DS:0 DS:1 DS:2 TOTAL
202302024-01-10T18:03:32.664Z New 0 0 0 0
202312024-01-10T18:03:32.664Z Sent 2 26 2 30
202322024-01-10T18:03:32.665Z Done 28 4 28 60
202332024-01-10T18:03:32.665Z Skipped 0 0 0 0
202342024-01-10T18:03:32.665Z Error 0 0 0 0
202352024-01-10T18:03:32.665ZLast Flush: 1016 0 1016
202362024-01-10T18:03:32.665ZDownstairs last five completed:
202372024-01-10T18:03:32.665ZUpstairs last five completed: 30 29 28 27 26
202382024-01-10T18:03:32.665ZWrote out file "/tmp/test_repair_verify.data"
202392024-01-10T18:03:32.670Z
202402024-01-10T18:03:32.674ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202412024-01-10T18:03:32.701ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202422024-01-10T18:03:32.702Z 0 000-019 186 186 186 346 346 346 F F F
202432024-01-10T18:03:32.702Z 1 020-039 186 186 186 346 346 346 F F F
202442024-01-10T18:03:32.702Z 2 040-059 186 186 186 346 346 346 F F F
202452024-01-10T18:03:32.702Z 3 060-079 184 184 184 343 343 343 F F F
202462024-01-10T18:03:32.702Z 4 080-099 186 186 186 346 346 346 T F T <---
202472024-01-10T18:03:32.709Z 5 100-119 188 184 188 348 343 348 F F F <---
202482024-01-10T18:03:32.709Z 6 120-139 184 184 184 343 343 343 F F F
202492024-01-10T18:03:32.709Z 7 140-159 184 184 184 344 344 344 F F F
202502024-01-10T18:03:32.709Z 8 160-179 184 184 184 344 344 344 F F F
202512024-01-10T18:03:32.709Z 9 180-199 185 185 185 345 345 345 F F F
202522024-01-10T18:03:32.709Z 10 200-219 186 186 186 346 346 346 F F F
202532024-01-10T18:03:32.709Z 11 220-239 186 186 186 346 346 346 T F T <---
202542024-01-10T18:03:32.709Z 12 240-259 188 187 188 349 347 349 T F T <---
202552024-01-10T18:03:32.709Z 13 260-279 182 182 182 337 337 337 F F F
202562024-01-10T18:03:32.709Z 14 280-299 182 182 182 338 338 338 T F T <---
202572024-01-10T18:03:32.709Z 15 300-319 188 182 188 350 338 350 T F T <---
202582024-01-10T18:03:32.709Z 16 320-339 188 186 188 348 346 348 F F F <---
202592024-01-10T18:03:32.710Z 17 340-359 188 181 188 348 335 348 F F F <---
202602024-01-10T18:03:32.710Z 18 360-379 181 181 181 335 335 335 T F T <---
202612024-01-10T18:03:32.710Z 19 380-399 182 182 182 340 340 340 F F F
202622024-01-10T18:03:32.710Z 20 400-419 188 187 188 348 347 348 F T F <---
202632024-01-10T18:03:32.710Z 21 420-439 188 181 188 350 335 350 F F F <---
202642024-01-10T18:03:32.710Z 22 440-459 188 187 188 350 347 350 F F F <---
202652024-01-10T18:03:32.710Z 23 460-479 187 187 187 347 347 347 T F T <---
202662024-01-10T18:03:32.710Z 24 480-499 182 182 182 338 338 338 F F F
202672024-01-10T18:03:32.710Z 25 500-519 182 182 182 340 340 340 T F T <---
202682024-01-10T18:03:32.710Z 26 520-539 188 181 188 349 335 349 F F F <---
202692024-01-10T18:03:32.710Z 27 540-559 186 186 186 346 346 346 F F F
202702024-01-10T18:03:32.710Z 28 560-579 186 186 186 346 346 346 F F F
202712024-01-10T18:03:32.710Z 29 580-599 188 185 188 348 345 348 F T F <---
202722024-01-10T18:03:32.710ZMax gen: 188, Max flush: 350
202732024-01-10T18:03:32.711ZError: Difference in extent metadata found!
202742024-01-10T18:03:32.711ZA Difference in extent metadata is expected here
202752024-01-10T18:03:32.711ZOn loop 94
202762024-01-10T18:03:32.711Z
202772024-01-10T18:03:32.717ZVerifying data now
202782024-01-10T18:03:34.267ZNo change after verify
202792024-01-10T18:03:34.268ZLoop: 94 Downstairs dump after verify (and repair):
202802024-01-10T18:03:34.285ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202812024-01-10T18:03:34.285Z 0 000-019 186 186 186 346 346 346 F F F
202822024-01-10T18:03:34.285Z 1 020-039 186 186 186 346 346 346 F F F
202832024-01-10T18:03:34.285Z 2 040-059 186 186 186 346 346 346 F F F
202842024-01-10T18:03:34.286Z 3 060-079 184 184 184 343 343 343 F F F
202852024-01-10T18:03:34.287Z 4 080-099 189 189 189 351 351 351 F F F
202862024-01-10T18:03:34.287Z 5 100-119 188 188 188 348 348 348 F F F
202872024-01-10T18:03:34.293Z 6 120-139 184 184 184 343 343 343 F F F
202882024-01-10T18:03:34.293Z 7 140-159 184 184 184 344 344 344 F F F
202892024-01-10T18:03:34.294Z 8 160-179 184 184 184 344 344 344 F F F
202902024-01-10T18:03:34.294Z 9 180-199 185 185 185 345 345 345 F F F
202912024-01-10T18:03:34.294Z 10 200-219 186 186 186 346 346 346 F F F
202922024-01-10T18:03:34.294Z 11 220-239 189 189 189 351 351 351 F F F
202932024-01-10T18:03:34.294Z 12 240-259 189 189 189 351 351 351 F F F
202942024-01-10T18:03:34.294Z 13 260-279 182 182 182 337 337 337 F F F
202952024-01-10T18:03:34.294Z 14 280-299 189 189 189 351 351 351 F F F
202962024-01-10T18:03:34.295Z 15 300-319 189 189 189 351 351 351 F F F
202972024-01-10T18:03:34.295Z 16 320-339 188 188 188 348 348 348 F F F
202982024-01-10T18:03:34.295Z 17 340-359 188 188 188 348 348 348 F F F
202992024-01-10T18:03:34.295Z 18 360-379 189 189 189 351 351 351 F F F
203002024-01-10T18:03:34.295Z 19 380-399 182 182 182 340 340 340 F F F
203012024-01-10T18:03:34.295Z 20 400-419 188 188 188 348 348 348 F F F
203022024-01-10T18:03:34.295Z 21 420-439 188 188 188 350 350 350 F F F
203032024-01-10T18:03:34.295Z 22 440-459 188 188 188 350 350 350 F F F
203042024-01-10T18:03:34.296Z 23 460-479 189 189 189 351 351 351 F F F
203052024-01-10T18:03:34.296Z 24 480-499 182 182 182 338 338 338 F F F
203062024-01-10T18:03:34.296Z 25 500-519 189 189 189 351 351 351 F F F
203072024-01-10T18:03:34.296Z 26 520-539 188 188 188 349 349 349 F F F
203082024-01-10T18:03:34.296Z 27 540-559 186 186 186 346 346 346 F F F
203092024-01-10T18:03:34.296Z 28 560-579 186 186 186 346 346 346 F F F
203102024-01-10T18:03:34.296Z 29 580-599 188 188 188 348 348 348 F F F
203112024-01-10T18:03:34.297ZMax gen: 189, Max flush: 351
203122024-01-10T18:03:34.297Z
203132024-01-10T18:03:34.297ZBegin loop 95
203142024-01-10T18:03:34.297ZDownstairs to restart: 1
203152024-01-10T18:03:34.315Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.310033559Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918}
203162024-01-10T18:03:34.324Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.310683295Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918}
203172024-01-10T18:03:34.324Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.31083135Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918}
203182024-01-10T18:03:34.325Z{Crucible runtime is spawned
203192024-01-10T18:03:34.325Z"msg":"Crucible 4203d854-c061-4382-b02d-8b33e246312e has session id: e88d9e69-3d1a-454f-9d64-9f70507e99f4","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311048676Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203202024-01-10T18:03:34.325Z{"msg":"Upstairs opts: Upstairs UUID: 4203d854-c061-4382-b02d-8b33e246312e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311132611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203212024-01-10T18:03:34.325Z{"msg":"Crucible stats registered with UUID: 4203d854-c061-4382-b02d-8b33e246312e","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.31116608Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203222024-01-10T18:03:34.325Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311287945Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918}
203232024-01-10T18:03:34.325Z{"msg":"4203d854-c061-4382-b02d-8b33e246312e active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311348244Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203242024-01-10T18:03:34.325Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311389229Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203252024-01-10T18:03:34.325Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311452551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203262024-01-10T18:03:34.325Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311495011Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203272024-01-10T18:03:34.325Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311575415Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203282024-01-10T18:03:34.326Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311620791Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203292024-01-10T18:03:34.326Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.31168527Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203302024-01-10T18:03:34.326Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.31172899Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203312024-01-10T18:03:34.326Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311806964Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203322024-01-10T18:03:34.326Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311900282Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203332024-01-10T18:03:34.327Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.311946393Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203342024-01-10T18:03:34.327Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.312041621Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203352024-01-10T18:03:34.327Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.31209058Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203362024-01-10T18:03:34.327Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.313230766Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203372024-01-10T18:03:34.327Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.313289255Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203382024-01-10T18:03:34.327Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.313425462Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203392024-01-10T18:03:34.327Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.313509121Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203402024-01-10T18:03:34.328Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.314173365Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203412024-01-10T18:03:34.328Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.314283246Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203422024-01-10T18:03:34.328Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.314324723Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203432024-01-10T18:03:34.328Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.314456418Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203442024-01-10T18:03:34.328Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.315297039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203452024-01-10T18:03:34.328Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.315400658Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203462024-01-10T18:03:34.328Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.315449204Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203472024-01-10T18:03:34.328Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.315490229Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203482024-01-10T18:03:34.329Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.315539432Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203492024-01-10T18:03:34.329Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.315597322Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203502024-01-10T18:03:34.329Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.315622582Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203512024-01-10T18:03:34.336Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.31623642Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203522024-01-10T18:03:34.337Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316311434Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203532024-01-10T18:03:34.337Z{"msg":"[0]R flush_numbers[0..12]: [346, 346, 346, 343, 351, 348, 343, 344, 344, 345, 346, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.31634708Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203542024-01-10T18:03:34.337Z{"msg":"[0]R generation[0..12]: [186, 186, 186, 184, 189, 188, 184, 184, 184, 185, 186, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316403465Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203552024-01-10T18:03:34.337Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316437122Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203562024-01-10T18:03:34.337Z{"msg":"[1]R flush_numbers[0..12]: [346, 346, 346, 343, 351, 348, 343, 344, 344, 345, 346, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316502345Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203572024-01-10T18:03:34.337Z{"msg":"[1]R generation[0..12]: [186, 186, 186, 184, 189, 188, 184, 184, 184, 185, 186, 189]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
203582024-01-10T18:03:34.337Z30,"time":"2024-01-10T18:03:34.316535668Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203592024-01-10T18:03:34.337Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316593452Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203602024-01-10T18:03:34.337Z{"msg":"[2]R flush_numbers[0..12]: [346, 346, 346, 343, 351, 348, 343, 344, 344, 345, 346, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316620817Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203612024-01-10T18:03:34.337Z{"msg":"[2]R generation[0..12]: [186, 186, 186, 184, 189, 188, 184, 184, 184, 185, 186, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.31667847Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203622024-01-10T18:03:34.337Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316706442Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203632024-01-10T18:03:34.337Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316768699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203642024-01-10T18:03:34.338Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316798244Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203652024-01-10T18:03:34.338Z{"msg":"Next flush: 352","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316854045Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203662024-01-10T18:03:34.338Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316879743Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203672024-01-10T18:03:34.338Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316913318Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203682024-01-10T18:03:34.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.31696707Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203692024-01-10T18:03:34.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.316992453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203702024-01-10T18:03:34.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.317021855Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203712024-01-10T18:03:34.338Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.317081035Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203722024-01-10T18:03:34.338Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.317103936Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203732024-01-10T18:03:34.339Z{"msg":"4203d854-c061-4382-b02d-8b33e246312e is now active with session: e88d9e69-3d1a-454f-9d64-9f70507e99f4","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.317128759Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203742024-01-10T18:03:34.339Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.31718109Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918,"session_id":"e88d9e69-3d1a-454f-9d64-9f70507e99f4"}
203752024-01-10T18:03:34.339Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:34.317205092Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1918}
203762024-01-10T18:03:34.340ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
203772024-01-10T18:03:34.340ZLoading write count information from file "/tmp/test_repair_verify.data"
203782024-01-10T18:03:34.340ZRun Repair workload
203792024-01-10T18:03:34.340Z01/30 Write block 457 len 28672 data: 15 15 15 15 14 12 15
203802024-01-10T18:03:34.340Z02/30 Write block 395 len 20480 data: 12 12 11 10 13
203812024-01-10T18:03:34.340Z03/30 Flush
203822024-01-10T18:03:34.340Z04/30 Write block 173 len 28672 data: 12 14 11 10 13 13 13
203832024-01-10T18:03:34.340Z05/30 Write block 166 len 4096 data: 22
203842024-01-10T18:03:34.340Z06/30 Write block 9 len 16384 data: 11 13 14 11
203852024-01-10T18:03:34.340Z07/30 Read block 338 len 12288
203862024-01-10T18:03:34.340Z08/30 Read block 573 len 16384
203872024-01-10T18:03:34.349Z09/30 Flush
203882024-01-10T18:03:34.349Z10/30 Write block 251 len 32768 data: 12 12 11 9 9 13 16 18
203892024-01-10T18:03:34.350Z11/30 Read block 251 len 24576
203902024-01-10T18:03:34.350Z12/30 Read block 573 len 28672
203912024-01-10T18:03:34.350Z13/30 Read block 204 len 8192
203922024-01-10T18:03:34.350Z14/30 Read block 7 len 36864
203932024-01-10T18:03:34.361Z15/30 Read block 157 len 4096
203942024-01-10T18:03:34.378Z16/30 Write block 196 len 4096 data: 20
203952024-01-10T18:03:34.378Z17/30 Read block 304 len 40960
203962024-01-10T18:03:34.378Z18/30 Write block 471 len 12288 data: 10 10 10
203972024-01-10T18:03:34.379Z19/30 Read block 286 len 4096
203982024-01-10T18:03:34.390Z20/30 Write block 265 len 20480 data: 13 15 16 14 14
203992024-01-10T18:03:34.391Z21/30 Write block 93 len 4096 data: 14
204002024-01-10T18:03:34.391Z22/30 Read block 529 len 12288
204012024-01-10T18:03:34.391Z23/30 Write block 252 len 20480 data: 13 12 10 10 14
204022024-01-10T18:03:34.391Z24/30 Read block 289 len 8192
204032024-01-10T18:03:34.391Z25/30 Read block 421 len 28672
204042024-01-10T18:03:34.391Z26/30 Read block 551 len 36864
204052024-01-10T18:03:34.391Z27/30 Write block 540 len 12288 data: 20 22 13
204062024-01-10T18:03:34.391Z28/30 Write block 253 len 16384 data: 13 11 11 15
204072024-01-10T18:03:34.391Z29/30 Write block 348 len 40960 data: 11 11 10 11 14 11 12 10 12 13
204082024-01-10T18:03:34.392Z30/30 Write block 551 len 20480 data: 20 23 23 22 18
204092024-01-10T18:03:34.392Z----------------------------------------------------------------
204102024-01-10T18:03:34.392Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:21
204112024-01-10T18:03:34.392ZStates: Active Active Active
204122024-01-10T18:03:34.392ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204132024-01-10T18:03:34.392Z 10 Acked 1009 Write 8 Done Sent Done false
204142024-01-10T18:03:34.392Z 11 Acked 1010 Read 6 Done Sent Done false
204152024-01-10T18:03:34.392Z 12 Acked 1011 Read 7 Done Sent Done false
204162024-01-10T18:03:34.392Z 13 Acked 1012 Read 2 Done Sent Done false
204172024-01-10T18:03:34.392Z 14 Acked 1013 Read 9 Done Sent Done false
204182024-01-10T18:03:34.392Z 15 Acked 1014 Read 1 Done Sent Done false
204192024-01-10T18:03:34.393Z 16 Acked 1015 Write 1 Done Sent Done false
204202024-01-10T18:03:34.393Z 17 Acked 1016 Read 10 Done Sent Done false
204212024-01-10T18:03:34.393Z 18 Acked 1017 Write 3 Done Sent Done false
204222024-01-10T18:03:34.393Z 19 Acked 1018 Read 1 Done Sent Done false
204232024-01-10T18:03:34.393Z 20 Acked 1019 Write 5 Done Sent Done false
204242024-01-10T18:03:34.393Z 21 Acked 1020 Write 1 Done Sent Done false
204252024-01-10T18:03:34.393Z 22 Acked 1021 Read 3 Done Sent Done false
204262024-01-10T18:03:34.393Z 23 Acked 1022 Write 5 Done Sent Done false
204272024-01-10T18:03:34.394Z 24 Acked 1023 Read 2 Done Sent Done false
204282024-01-10T18:03:34.394Z 25 Acked 1024 Read 7 Done Sent Done false
204292024-01-10T18:03:34.394Z 26 Acked 1025 Read 9 Done Sent Done false
204302024-01-10T18:03:34.394Z 27 Acked 1026 Write 3 Done Sent Done false
204312024-01-10T18:03:34.394Z 28 Acked 1027 Write 4 Done Sent Done false
204322024-01-10T18:03:34.394Z 29 Acked 1028 Write 10 Sent Sent Sent false
204332024-01-10T18:03:34.394Z 30 Acked 1029 Write 5 Sent Sent Sent false
204342024-01-10T18:03:34.395Z STATES DS:0 DS:1 DS:2 TOTAL
204352024-01-10T18:03:34.395Z New 0 0 0 0
204362024-01-10T18:03:34.395Z Sent 2 21 2 25
204372024-01-10T18:03:34.395Z Done 19 0 19 38
204382024-01-10T18:03:34.395Z Skipped 0 0 0 0
204392024-01-10T18:03:34.395Z Error 0 0 0 0
204402024-01-10T18:03:34.395ZLast Flush: 1008 1008 1008
204412024-01-10T18:03:34.396ZDownstairs last five completed: 1008 1007 1006 1005 1004
204422024-01-10T18:03:34.396ZUpstairs last five completed: 30 29 28 27 26
204432024-01-10T18:03:34.397ZWrote out file "/tmp/test_repair_verify.data"
204442024-01-10T18:03:34.397Z
204452024-01-10T18:03:34.407ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
204462024-01-10T18:03:34.415ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204472024-01-10T18:03:34.415Z 0 000-019 190 190 190 353 353 353 F F F
204482024-01-10T18:03:34.415Z 1 020-039 186 186 186 346 346 346 F F F
204492024-01-10T18:03:34.415Z 2 040-059 186 186 186 346 346 346 F F F
204502024-01-10T18:03:34.416Z 3 060-079 184 184 184 343 343 343 F F F
204512024-01-10T18:03:34.416Z 4 080-099 189 189 189 351 351 351 T F T <---
204522024-01-10T18:03:34.416Z 5 100-119 188 188 188 348 348 348 F F F
204532024-01-10T18:03:34.425Z 6 120-139 184 184 184 343 343 343 F F F
204542024-01-10T18:03:34.425Z 7 140-159 184 184 184 344 344 344 F F F
204552024-01-10T18:03:34.425Z 8 160-179 190 190 190 353 353 353 F F F
204562024-01-10T18:03:34.425Z 9 180-199 185 185 185 345 345 345 T F T <---
204572024-01-10T18:03:34.425Z 10 200-219 186 186 186 346 346 346 F F F
204582024-01-10T18:03:34.426Z 11 220-239 189 189 189 351 351 351 F F F
204592024-01-10T18:03:34.426Z 12 240-259 189 189 189 351 351 351 T F T <---
204602024-01-10T18:03:34.426Z 13 260-279 182 182 182 337 337 337 T F T <---
204612024-01-10T18:03:34.426Z 14 280-299 189 189 189 351 351 351 F F F
204622024-01-10T18:03:34.426Z 15 300-319 189 189 189 351 351 351 F F F
204632024-01-10T18:03:34.426Z 16 320-339 188 188 188 348 348 348 F F F
204642024-01-10T18:03:34.426Z 17 340-359 188 188 188 348 348 348 T F T <---
204652024-01-10T18:03:34.426Z 18 360-379 189 189 189 351 351 351 F F F
204662024-01-10T18:03:34.427Z 19 380-399 190 190 190 352 352 352 F F F
204672024-01-10T18:03:34.427Z 20 400-419 188 188 188 348 348 348 F F F
204682024-01-10T18:03:34.427Z 21 420-439 188 188 188 350 350 350 F F F
204692024-01-10T18:03:34.427Z 22 440-459 190 190 190 352 352 352 F F F
204702024-01-10T18:03:34.427Z 23 460-479 190 190 190 352 352 352 T F T <---
204712024-01-10T18:03:34.427Z 24 480-499 182 182 182 338 338 338 F F F
204722024-01-10T18:03:34.427Z 25 500-519 189 189 189 351 351 351 F F F
204732024-01-10T18:03:34.427Z 26 520-539 188 188 188 349 349 349 F F F
204742024-01-10T18:03:34.427Z 27 540-559 186 186 186 346 346 346 T F T <---
204752024-01-10T18:03:34.428Z 28 560-579 186 186 186 346 346 346 F F F
204762024-01-10T18:03:34.428Z 29 580-599 188 188 188 348 348 348 F F F
204772024-01-10T18:03:34.428ZMax gen: 190, Max flush: 353
204782024-01-10T18:03:34.428ZError: Difference in extent metadata found!
204792024-01-10T18:03:34.428ZA Difference in extent metadata is expected here
204802024-01-10T18:03:34.428ZOn loop 95
204812024-01-10T18:03:34.428Z
204822024-01-10T18:03:34.428ZVerifying data now
204832024-01-10T18:03:39.262ZNo change after verify
204842024-01-10T18:03:39.262ZLoop: 95 Downstairs dump after verify (and repair):
204852024-01-10T18:03:39.273ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204862024-01-10T18:03:39.273Z 0 000-019 190 190 190 353 353 353 F F F
204872024-01-10T18:03:39.274Z 1 020-039 186 186 186 346 346 346 F F F
204882024-01-10T18:03:39.274Z 2 040-059 186 186 186 346 346 346 F F F
204892024-01-10T18:03:39.274Z 3 060-079 184 184 184 343 343 343 F F F
204902024-01-10T18:03:39.279Z 4 080-099 191 191 191 354 354 354 F F F
204912024-01-10T18:03:39.279Z 5 100-119 188 188 188 348 348 348 F F F
204922024-01-10T18:03:39.279Z 6 120-139 184 184 184 343 343 343 F F F
204932024-01-10T18:03:39.279Z 7 140-159 184 184 184 344 344 344 F F F
204942024-01-10T18:03:39.279Z 8 160-179 190 190 190 353 353 353 F F F
204952024-01-10T18:03:39.280Z 9 180-199 191 191 191 354 354 354 F F F
204962024-01-10T18:03:39.280Z 10 200-219 186 186 186 346 346 346 F F F
204972024-01-10T18:03:39.280Z 11 220-239 189 189 189 351 351 351 F F F
204982024-01-10T18:03:39.280Z 12 240-259 191 191 191 354 354 354 F F F
204992024-01-10T18:03:39.280Z 13 260-279 191 191 191 354 354 354 F F F
205002024-01-10T18:03:39.280Z 14 280-299 189 189 189 351 351 351 F F F
205012024-01-10T18:03:39.280Z 15 300-319 189 189 189 351 351 351 F F F
205022024-01-10T18:03:39.280Z 16 320-339 188 188 188 348 348 348 F F F
205032024-01-10T18:03:39.280Z 17 340-359 191 191 191 354 354 354 F F F
205042024-01-10T18:03:39.280Z 18 360-379 189 189 189 351 351 351 F F F
205052024-01-10T18:03:39.280Z 19 380-399 190 190 190 352 352 352 F F F
205062024-01-10T18:03:39.280Z 20 400-419 188 188 188 348 348 348 F F F
205072024-01-10T18:03:39.280Z 21 420-439 188 188 188 350 350 350 F F F
205082024-01-10T18:03:39.280Z 22 440-459 190 190 190 352 352 352 F F F
205092024-01-10T18:03:39.280Z 23 460-479 191 191 191 354 354 354 F F F
205102024-01-10T18:03:39.281Z 24 480-499 182 182 182 338 338 338 F F F
205112024-01-10T18:03:39.282Z 25 500-519 189 189 189 351 351 351 F F F
205122024-01-10T18:03:39.282Z 26 520-539 188 188 188 349 349 349 F F F
205132024-01-10T18:03:39.284Z 27 540-559 191 191 191 354 354 354 F F F
205142024-01-10T18:03:39.284Z 28 560-579 186 186 186 346 346 346 F F F
205152024-01-10T18:03:39.284Z 29 580-599 188 188 188 348 348 348 F F F
205162024-01-10T18:03:39.284ZMax gen: 191, Max flush: 354
205172024-01-10T18:03:39.284Z
205182024-01-10T18:03:39.284ZBegin loop 96
205192024-01-10T18:03:39.284ZDownstairs to restart: 1
205202024-01-10T18:03:39.306Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.302895522Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926}
205212024-01-10T18:03:39.306Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.303413933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926}
205222024-01-10T18:03:39.306Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.303492335Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926}
205232024-01-10T18:03:39.311Z{"msg":"Crucible 7aff91e1-a8eb-47ac-948e-bf4846c0076b has session id: 34b68e0d-2b9e-41e5-95d4-e72992f8f18e","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.303584404Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205242024-01-10T18:03:39.311Z{"msg":"Upstairs opts: Upstairs UUID: 7aff91e1-a8eb-47ac-948e-bf4846c0076b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.303680325Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205252024-01-10T18:03:39.311Z{Crucible runtime is spawned
205262024-01-10T18:03:39.311Z"msg":"Crucible stats registered with UUID: 7aff91e1-a8eb-47ac-948e-bf4846c0076b","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.303821039Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205272024-01-10T18:03:39.311Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.303882032Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926}
205282024-01-10T18:03:39.312Z{"msg":"7aff91e1-a8eb-47ac-948e-bf4846c0076b active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304033963Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205292024-01-10T18:03:39.312Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304095895Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205302024-01-10T18:03:39.312Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304122912Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205312024-01-10T18:03:39.312Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304250592Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205322024-01-10T18:03:39.312Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304335178Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205332024-01-10T18:03:39.312Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304378877Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205342024-01-10T18:03:39.312Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304478697Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205352024-01-10T18:03:39.312Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304587177Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205362024-01-10T18:03:39.312Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304626938Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205372024-01-10T18:03:39.312Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304847592Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205382024-01-10T18:03:39.312Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304941117Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205392024-01-10T18:03:39.312Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.304973663Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205402024-01-10T18:03:39.312Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.305198694Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205412024-01-10T18:03:39.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.305936025Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205422024-01-10T18:03:39.313Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.306012548Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205432024-01-10T18:03:39.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.306040465Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205442024-01-10T18:03:39.313Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.30617935Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205452024-01-10T18:03:39.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.306225627Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205462024-01-10T18:03:39.313Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.306265581Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205472024-01-10T18:03:39.314Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.306857966Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205482024-01-10T18:03:39.314Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.307029704Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205492024-01-10T18:03:39.321Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.307485443Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205502024-01-10T18:03:39.322Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.307791422Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205512024-01-10T18:03:39.322Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.307834259Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205522024-01-10T18:03:39.322Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.307858004Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205532024-01-10T18:03:39.322Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.307901421Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205542024-01-10T18:03:39.322Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.307944773Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205552024-01-10T18:03:39.322Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308004171Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205562024-01-10T18:03:39.322Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308187857Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205572024-01-10T18:03:39.323Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308309554Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205582024-01-10T18:03:39.323Z{"msg":"[0]R flush_numbers[0..12]: [353, 346, 346, 343, 354, 348, 343, 344, 353, 354, 346, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308380742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205592024-01-10T18:03:39.323Z{"msg":"[0]R generation[0..12]: [190, 186, 186, 184, 191, 188, 184, 184, 190, 191, 186, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308409522Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205602024-01-10T18:03:39.323Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308455661Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205612024-01-10T18:03:39.323Z{"msg":"[1]R flush_numbers[0..12]: [353, 346, 346, 343, 354, 348, 343, 344, 353, 354, 346, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308480976Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205622024-01-10T18:03:39.323Z{"msg":"[1]R generation[0..12]: [190, 186, 186, 184, 191, 188, 184, 184, 190, 191, 186, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308504072Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205632024-01-10T18:03:39.323Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308546423Z","Wait for a query_work_queue command to finish before sending IO
205642024-01-10T18:03:39.323Zhostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205652024-01-10T18:03:39.324Z{"msg":"[2]R flush_numbers[0..12]: [353, 346, 346, 343, 354, 348, 343, 344, 353, 354, 346, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308575407Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205662024-01-10T18:03:39.324Z{"msg":"[2]R generation[0..12]: [190, 186, 186, 184, 191, 188, 184, 184, 190, 191, 186, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308621887Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205672024-01-10T18:03:39.324Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308647115Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205682024-01-10T18:03:39.325Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308668742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205692024-01-10T18:03:39.325Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308708673Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205702024-01-10T18:03:39.325Z{"msg":"Next flush: 355","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308731433Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205712024-01-10T18:03:39.325Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308754155Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205722024-01-10T18:03:39.325Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308775885Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205732024-01-10T18:03:39.325Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308795582Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205742024-01-10T18:03:39.325Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308818226Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205752024-01-10T18:03:39.325Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308839983Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205762024-01-10T18:03:39.325Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308862832Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205772024-01-10T18:03:39.326Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308920269Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205782024-01-10T18:03:39.326Z{"msg":"7aff91e1-a8eb-47ac-948e-bf4846c0076b is now active with session: 34b68e0d-2b9e-41e5-95d4-e72992f8f18e","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308945723Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205792024-01-10T18:03:39.326Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.308999091Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926,"session_id":"34b68e0d-2b9e-41e5-95d4-e72992f8f18e"}
205802024-01-10T18:03:39.326Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:39.309022236Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1926}
205812024-01-10T18:03:39.326ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
205822024-01-10T18:03:39.326ZLoading write count information from file "/tmp/test_repair_verify.data"
205832024-01-10T18:03:39.326ZRun Repair workload
205842024-01-10T18:03:39.326Z01/30 Flush
205852024-01-10T18:03:39.326Z02/30 Write block 419 len 4096 data: 19
205862024-01-10T18:03:39.326Z03/30 Read block 487 len 40960
205872024-01-10T18:03:39.330Z04/30 Write block 57 len 8192 data: 14 18
205882024-01-10T18:03:39.330Z05/30 Read block 472 len 12288
205892024-01-10T18:03:39.330Z06/30 Read block 0 len 12288
205902024-01-10T18:03:39.330Z07/30 Write block 170 len 32768 data: 14 13 13 13 15 12 11 14
205912024-01-10T18:03:39.330Z08/30 Read block 49 len 32768
205922024-01-10T18:03:39.330Z09/30 Flush
205932024-01-10T18:03:39.333Z10/30 Write block 200 len 16384 data: 19 18 16 13
205942024-01-10T18:03:39.333Z11/30 Write block 243 len 12288 data: 19 24 22
205952024-01-10T18:03:39.333Z12/30 Write block 36 len 40960 data: 16 15 13 12 11 14 15 13 15 15
205962024-01-10T18:03:39.339Z13/30 Write block 140 len 36864 data: 15 16 16 14 14 14 15 17 16
205972024-01-10T18:03:39.339Z14/30 Read block 533 len 20480
205982024-01-10T18:03:39.339Z15/30 Write block 591 len 8192 data: 12 11
205992024-01-10T18:03:39.339Z16/30 Read block 383 len 20480
206002024-01-10T18:03:39.343Z17/30 Write block 242 len 32768 data: 16 20 25 23 16 14 14 15
206012024-01-10T18:03:39.346Z18/30 Write block 5 len 40960 data: 10 10 9 9 12 14 15 12 11 13
206022024-01-10T18:03:39.346Z19/30 Read block 587 len 32768
206032024-01-10T18:03:39.349Z20/30 Write block 509 len 36864 data: 18 17 20 20 18 20 19 16 15
206042024-01-10T18:03:39.350Z21/30 Read block 87 len 36864
206052024-01-10T18:03:39.353Z22/30 Read block 566 len 16384
206062024-01-10T18:03:39.357Z23/30 Read block 487 len 36864
206072024-01-10T18:03:39.357Z24/30 Read block 174 len 12288
206082024-01-10T18:03:39.360Z25/30 Write block 31 len 4096 data: 14
206092024-01-10T18:03:39.361Z26/30 Write block 569 len 8192 data: 14 14
206102024-01-10T18:03:39.361Z27/30 Read block 255 len 28672
206112024-01-10T18:03:39.366Z28/30 Write block 163 len 40960 data: 17 19 21 23 20 18 17 15 14 14
206122024-01-10T18:03:39.369Z29/30 Write block 390 len 12288 data: 12 12 13
206132024-01-10T18:03:39.369Z30/30 Write block 544 len 36864 data: 14 16 15 17 18 19 21 21 24
206142024-01-10T18:03:39.369Z----------------------------------------------------------------
206152024-01-10T18:03:39.369Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:29
206162024-01-10T18:03:39.370ZStates: Active Active Active
206172024-01-10T18:03:39.370ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206182024-01-10T18:03:39.370Z 2 Acked 1001 Write 1 Done Sent Done false
206192024-01-10T18:03:39.370Z 3 Acked 1002 Read 10 Done Sent Done false
206202024-01-10T18:03:39.370Z 4 Acked 1003 Write 2 Done Sent Done false
206212024-01-10T18:03:39.370Z 5 Acked 1004 Read 3 Done Sent Done false
206222024-01-10T18:03:39.370Z 6 Acked 1005 Read 3 Done Sent Done false
206232024-01-10T18:03:39.370Z 7 Acked 1006 Write 8 Done Sent Done false
206242024-01-10T18:03:39.370Z 8 Acked 1007 Read 8 Done Sent Done false
206252024-01-10T18:03:39.371Z 9 Acked 1008 Flush 0 Done Sent Done false
206262024-01-10T18:03:39.371Z 10 Acked 1009 Write 4 Done Sent Done false
206272024-01-10T18:03:39.371Z 11 Acked 1010 Write 3 Done Sent Done false
206282024-01-10T18:03:39.371Z 12 Acked 1011 Write 10 Done Sent Done false
206292024-01-10T18:03:39.371Z 13 Acked 1012 Write 9 Done Sent Done false
206302024-01-10T18:03:39.371Z 14 Acked 1013 Read 5 Done Sent Done false
206312024-01-10T18:03:39.371Z 15 Acked 1014 Write 2 Done Sent Done false
206322024-01-10T18:03:39.371Z 16 Acked 1015 Read 5 Done Sent Done false
206332024-01-10T18:03:39.371Z 17 Acked 1016 Write 8 Done Sent Done false
206342024-01-10T18:03:39.371Z 18 Acked 1017 Write 10 Done Sent Done false
206352024-01-10T18:03:39.371Z 19 Acked 1018 Read 8 Done Sent Done false
206362024-01-10T18:03:39.371Z 20 Acked 1019 Write 9 Done Sent Done false
206372024-01-10T18:03:39.371Z 21 Acked 1020 Read 9 Done Sent Done false
206382024-01-10T18:03:39.371Z 22 Acked 1021 Read 4 Done Sent Done false
206392024-01-10T18:03:39.371Z 23 Acked 1022 Read 9 Done Sent Done false
206402024-01-10T18:03:39.371Z 24 Acked 1023 Read 3 Done Sent Done false
206412024-01-10T18:03:39.371Z 25 Acked 1024 Write 1 Done Sent Done false
206422024-01-10T18:03:39.371Z 26 Acked 1025 Write 2 Done Sent Done false
206432024-01-10T18:03:39.374Z 27 Acked 1026 Read 7 Done Sent Done false
206442024-01-10T18:03:39.374Z 28 Acked 1027 Write 10 Done Sent Done false
206452024-01-10T18:03:39.374Z 29 Acked 1028 Write 3 Done Sent Done false
206462024-01-10T18:03:39.374Z 30 Acked 1029 Write 9 Sent Sent Sent false
206472024-01-10T18:03:39.374Z STATES DS:0 DS:1 DS:2 TOTAL
206482024-01-10T18:03:39.374Z New 0 0 0 0
206492024-01-10T18:03:39.374Z Sent 1 29 1 31
206502024-01-10T18:03:39.374Z Done 28 0 28 56
206512024-01-10T18:03:39.374Z Skipped 0 0 0 0
206522024-01-10T18:03:39.374Z Error 0 0 0 0
206532024-01-10T18:03:39.374ZLast Flush: 1008 1000 1008
206542024-01-10T18:03:39.374ZDownstairs last five completed: 1000
206552024-01-10T18:03:39.374ZUpstairs last five completed: 30 29 28 27 26
206562024-01-10T18:03:39.374ZWrote out file "/tmp/test_repair_verify.data"
206572024-01-10T18:03:39.377Z
206582024-01-10T18:03:39.381ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
206592024-01-10T18:03:39.406ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206602024-01-10T18:03:39.406Z 0 000-019 190 190 190 353 353 353 T F T <---
206612024-01-10T18:03:39.406Z 1 020-039 186 186 186 346 346 346 T F T <---
206622024-01-10T18:03:39.407Z 2 040-059 192 186 192 356 346 356 T F T <---
206632024-01-10T18:03:39.407Z 3 060-079 184 184 184 343 343 343 F F F
206642024-01-10T18:03:39.407Z 4 080-099 191 191 191 354 354 354 F F F
206652024-01-10T18:03:39.411Z 5 100-119 188 188 188 348 348 348 F F F
206662024-01-10T18:03:39.411Z 6 120-139 184 184 184 343 343 343 F F F
206672024-01-10T18:03:39.411Z 7 140-159 184 184 184 344 344 344 T F T <---
206682024-01-10T18:03:39.411Z 8 160-179 192 190 192 356 353 356 T F T <---
206692024-01-10T18:03:39.411Z 9 180-199 191 191 191 354 354 354 F F F
206702024-01-10T18:03:39.411Z 10 200-219 186 186 186 346 346 346 T F T <---
206712024-01-10T18:03:39.411Z 11 220-239 189 189 189 351 351 351 F F F
206722024-01-10T18:03:39.411Z 12 240-259 191 191 191 354 354 354 T F T <---
206732024-01-10T18:03:39.411Z 13 260-279 191 191 191 354 354 354 F F F
206742024-01-10T18:03:39.411Z 14 280-299 189 189 189 351 351 351 F F F
206752024-01-10T18:03:39.411Z 15 300-319 189 189 189 351 351 351 F F F
206762024-01-10T18:03:39.412Z 16 320-339 188 188 188 348 348 348 F F F
206772024-01-10T18:03:39.412Z 17 340-359 191 191 191 354 354 354 F F F
206782024-01-10T18:03:39.412Z 18 360-379 189 189 189 351 351 351 F F F
206792024-01-10T18:03:39.412Z 19 380-399 190 190 190 352 352 352 T F T <---
206802024-01-10T18:03:39.412Z 20 400-419 192 188 192 356 348 356 F F F <---
206812024-01-10T18:03:39.412Z 21 420-439 188 188 188 350 350 350 F F F
206822024-01-10T18:03:39.412Z 22 440-459 190 190 190 352 352 352 F F F
206832024-01-10T18:03:39.412Z 23 460-479 191 191 191 354 354 354 F F F
206842024-01-10T18:03:39.412Z 24 480-499 182 182 182 338 338 338 F F F
206852024-01-10T18:03:39.412Z 25 500-519 189 189 189 351 351 351 T F T <---
206862024-01-10T18:03:39.412Z 26 520-539 188 188 188 349 349 349 F F F
206872024-01-10T18:03:39.412Z 27 540-559 191 191 191 354 354 354 T F T <---
206882024-01-10T18:03:39.412Z 28 560-579 186 186 186 346 346 346 T F T <---
206892024-01-10T18:03:39.412Z 29 580-599 188 188 188 348 348 348 T F T <---
206902024-01-10T18:03:39.412ZMax gen: 192, Max flush: 356
206912024-01-10T18:03:39.412ZError: Difference in extent metadata found!
206922024-01-10T18:03:39.432ZA Difference in extent metadata is expected here
206932024-01-10T18:03:39.433ZOn loop 96
206942024-01-10T18:03:39.433Z
206952024-01-10T18:03:39.433ZVerifying data now
206962024-01-10T18:03:44.704ZNo change after verify
206972024-01-10T18:03:44.704ZLoop: 96 Downstairs dump after verify (and repair):
206982024-01-10T18:03:44.726ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206992024-01-10T18:03:44.726Z 0 000-019 193 193 193 357 357 357 F F F
207002024-01-10T18:03:44.726Z 1 020-039 193 193 193 357 357 357 F F F
207012024-01-10T18:03:44.726Z 2 040-059 193 193 193 357 357 357 F F F
207022024-01-10T18:03:44.726Z 3 060-079 184 184 184 343 343 343 F F F
207032024-01-10T18:03:44.726Z 4 080-099 191 191 191 354 354 354 F F F
207042024-01-10T18:03:44.726Z 5 100-119 188 188 188 348 348 348 F F F
207052024-01-10T18:03:44.730Z 6 120-139 184 184 184 343 343 343 F F F
207062024-01-10T18:03:44.730Z 7 140-159 193 193 193 357 357 357 F F F
207072024-01-10T18:03:44.730Z 8 160-179 193 193 193 357 357 357 F F F
207082024-01-10T18:03:44.730Z 9 180-199 191 191 191 354 354 354 F F F
207092024-01-10T18:03:44.730Z 10 200-219 193 193 193 357 357 357 F F F
207102024-01-10T18:03:44.730Z 11 220-239 189 189 189 351 351 351 F F F
207112024-01-10T18:03:44.730Z 12 240-259 193 193 193 357 357 357 F F F
207122024-01-10T18:03:44.731Z 13 260-279 191 191 191 354 354 354 F F F
207132024-01-10T18:03:44.731Z 14 280-299 189 189 189 351 351 351 F F F
207142024-01-10T18:03:44.731Z 15 300-319 189 189 189 351 351 351 F F F
207152024-01-10T18:03:44.731Z 16 320-339 188 188 188 348 348 348 F F F
207162024-01-10T18:03:44.731Z 17 340-359 191 191 191 354 354 354 F F F
207172024-01-10T18:03:44.731Z 18 360-379 189 189 189 351 351 351 F F F
207182024-01-10T18:03:44.731Z 19 380-399 193 193 193 357 357 357 F F F
207192024-01-10T18:03:44.731Z 20 400-419 192 192 192 356 356 356 F F F
207202024-01-10T18:03:44.731Z 21 420-439 188 188 188 350 350 350 F F F
207212024-01-10T18:03:44.731Z 22 440-459 190 190 190 352 352 352 F F F
207222024-01-10T18:03:44.731Z 23 460-479 191 191 191 354 354 354 F F F
207232024-01-10T18:03:44.731Z 24 480-499 182 182 182 338 338 338 F F F
207242024-01-10T18:03:44.731Z 25 500-519 193 193 193 357 357 357 F F F
207252024-01-10T18:03:44.731Z 26 520-539 188 188 188 349 349 349 F F F
207262024-01-10T18:03:44.731Z 27 540-559 193 193 193 357 357 357 F F F
207272024-01-10T18:03:44.731Z 28 560-579 193 193 193 357 357 357 F F F
207282024-01-10T18:03:44.731Z 29 580-599 193 193 193 357 357 357 F F F
207292024-01-10T18:03:44.731ZMax gen: 193, Max flush: 357
207302024-01-10T18:03:44.735Z
207312024-01-10T18:03:44.735ZBegin loop 97
207322024-01-10T18:03:44.735ZDownstairs to restart: 0
207332024-01-10T18:03:44.762Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.759083434Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934}
207342024-01-10T18:03:44.766Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.759565353Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934}
207352024-01-10T18:03:44.767Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.759636085Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934}
207362024-01-10T18:03:44.767Z{"msg":"Crucible 31d25c6d-e740-47ab-9045-d28408fc101e has session id: f137b999-06ee-4b64-b7ae-6a4dcbdb63b1","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.759676776Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207372024-01-10T18:03:44.767Z{"msg":"Upstairs opts: Upstairs UUID: 31d25c6d-e740-47ab-9045-d28408fc101e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.759763709Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207382024-01-10T18:03:44.767Z{"msg":"Crucible stats registered with UUID: 31d25c6d-e740-47ab-9045-d28408fc101e","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.759845017Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207392024-01-10T18:03:44.767ZCrucible runtime is spawned
207402024-01-10T18:03:44.768Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760109812Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934}
207412024-01-10T18:03:44.768Z{"msg":"31d25c6d-e740-47ab-9045-d28408fc101e active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760220821Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207422024-01-10T18:03:44.768Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760304217Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207432024-01-10T18:03:44.768Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760349399Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207442024-01-10T18:03:44.768Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760446004Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207452024-01-10T18:03:44.768Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760544664Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207462024-01-10T18:03:44.769Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760612551Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207472024-01-10T18:03:44.769Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760655554Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207482024-01-10T18:03:44.769Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760694768Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207492024-01-10T18:03:44.769Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760738203Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207502024-01-10T18:03:44.769Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760825266Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207512024-01-10T18:03:44.769Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760867587Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207522024-01-10T18:03:44.769Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.760985425Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207532024-01-10T18:03:44.769Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.761161924Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207542024-01-10T18:03:44.769Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.761901821Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207552024-01-10T18:03:44.769Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.761964406Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207562024-01-10T18:03:44.769Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.762260195Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207572024-01-10T18:03:44.769Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.762343564Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207582024-01-10T18:03:44.774Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.763065363Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207592024-01-10T18:03:44.774Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.763384239Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207602024-01-10T18:03:44.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.764025272Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207612024-01-10T18:03:44.775Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.764088592Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207622024-01-10T18:03:44.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.764114906Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207632024-01-10T18:03:44.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.764255588Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207642024-01-10T18:03:44.775Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.764289793Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207652024-01-10T18:03:44.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.764313191Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207662024-01-10T18:03:44.775Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.764357233Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207672024-01-10T18:03:44.776Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.764381403Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207682024-01-10T18:03:44.776Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.765280693Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207692024-01-10T18:03:44.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.765989884Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207702024-01-10T18:03:44.776Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766065474Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207712024-01-10T18:03:44.776Z{"msg":"[0]R flush_numbers[0..12]: [357, 357, 357, 343, 354, 348, 343, 357, 357, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766143073Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207722024-01-10T18:03:44.776Z{"msg":"[0]R generation[0..12]: [193, 193, 193, 184, 191, 188, 184, 193, 193, 191, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766212131Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207732024-01-10T18:03:44.776Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766274303Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207742024-01-10T18:03:44.776Z{"msg":"[1]R flush_numbers[0..12]: [357, 357, 357, 343, 354, 348, 343, 357, 357, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766309451Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207752024-01-10T18:03:44.777Z{"msg":"[1]R generation[0..12]: [193, 193, 193, 184, 191, 188, 184, 193, 193, 191, 193, 189]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
207762024-01-10T18:03:44.777Z,"time":"2024-01-10T18:03:44.766383482Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207772024-01-10T18:03:44.777Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766451814Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207782024-01-10T18:03:44.777Z{"msg":"[2]R flush_numbers[0..12]: [357, 357, 357, 343, 354, 348, 343, 357, 357, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766485598Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207792024-01-10T18:03:44.777Z{"msg":"[2]R generation[0..12]: [193, 193, 193, 184, 191, 188, 184, 193, 193, 191, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766542969Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207802024-01-10T18:03:44.777Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766570699Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207812024-01-10T18:03:44.777Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766628485Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207822024-01-10T18:03:44.777Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766655445Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207832024-01-10T18:03:44.777Z{"msg":"Next flush: 358","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.7667191Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207842024-01-10T18:03:44.777Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766750517Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207852024-01-10T18:03:44.778Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766814135Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207862024-01-10T18:03:44.778Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766842136Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207872024-01-10T18:03:44.778Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766898611Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207882024-01-10T18:03:44.779Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766927998Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207892024-01-10T18:03:44.779Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.766983827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207902024-01-10T18:03:44.779Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
207912024-01-10T18:03:44.779ZSet Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.767070173Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207922024-01-10T18:03:44.779Z{"msg":"31d25c6d-e740-47ab-9045-d28408fc101e is now active with session: f137b999-06ee-4b64-b7ae-6a4dcbdb63b1","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.76712964Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207932024-01-10T18:03:44.779Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.767159063Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934,"session_id":"f137b999-06ee-4b64-b7ae-6a4dcbdb63b1"}
207942024-01-10T18:03:44.779Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:44.767213996Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1934}
207952024-01-10T18:03:44.779ZLoading write count information from file "/tmp/test_repair_verify.data"
207962024-01-10T18:03:44.779ZRun Repair workload
207972024-01-10T18:03:44.779Z01/30 Write block 303 len 28672 data: 15 13 12 11 9 7 7
207982024-01-10T18:03:44.782Z02/30 Read block 276 len 16384
207992024-01-10T18:03:44.782Z03/30 Flush
208002024-01-10T18:03:44.782Z04/30 Write block 409 len 12288 data: 16 16 17
208012024-01-10T18:03:44.782Z05/30 Read block 8 len 20480
208022024-01-10T18:03:44.783Z06/30 Write block 331 len 36864 data: 10 8 6 6 14 13 13 13 16
208032024-01-10T18:03:44.786Z07/30 Write block 375 len 12288 data: 13 11 11
208042024-01-10T18:03:44.786Z08/30 Write block 98 len 32768 data: 13 16 15 10 10 10 6 8
208052024-01-10T18:03:44.786Z09/30 Write block 559 len 40960 data: 12 13 15 16 17 16 16 13 14 15
208062024-01-10T18:03:44.789Z10/30 Write block 145 len 12288 data: 15 16 18
208072024-01-10T18:03:44.789Z11/30 Write block 300 len 40960 data: 17 16 16 16 14 13 12 10 8 8
208082024-01-10T18:03:44.792Z12/30 Write block 322 len 40960 data: 10 13 15 15 13 12 11 9 8 11
208092024-01-10T18:03:44.792Z13/30 Read block 194 len 24576
208102024-01-10T18:03:44.795Z14/30 Read block 108 len 24576
208112024-01-10T18:03:44.798Z15/30 Read block 396 len 36864
208122024-01-10T18:03:44.801Z16/30 Read block 214 len 36864
208132024-01-10T18:03:44.804Z17/30 Write block 589 len 8192 data: 11 11
208142024-01-10T18:03:44.804Z18/30 Read block 366 len 28672
208152024-01-10T18:03:44.807Z19/30 Write block 569 len 28672 data: 15 15 14 14 15 18 17
208162024-01-10T18:03:44.810Z20/30 Write block 72 len 8192 data: 7 8
208172024-01-10T18:03:44.810Z21/30 Read block 465 len 40960
208182024-01-10T18:03:44.814Z22/30 Read block 73 len 24576
208192024-01-10T18:03:44.817Z23/30 Read block 254 len 20480
208202024-01-10T18:03:44.817Z24/30 Write block 582 len 12288 data: 16 16 13
208212024-01-10T18:03:44.820Z25/30 Read block 328 len 4096
208222024-01-10T18:03:44.821Z26/30 Read block 403 len 8192
208232024-01-10T18:03:44.821Z27/30 Read block 473 len 20480
208242024-01-10T18:03:44.824Z28/30 Read block 487 len 16384
208252024-01-10T18:03:44.824Z29/30 Read block 277 len 32768
208262024-01-10T18:03:44.828Z30/30 Read block 205 len 8192
208272024-01-10T18:03:44.832Z----------------------------------------------------------------
208282024-01-10T18:03:44.832Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:27
208292024-01-10T18:03:44.832ZStates: Active Active Active
208302024-01-10T18:03:44.832ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208312024-01-10T18:03:44.832Z 4 Acked 1003 Write 3 Done Done Done false
208322024-01-10T18:03:44.832Z 5 Acked 1004 Read 5 Done Done Done false
208332024-01-10T18:03:44.832Z 6 Acked 1005 Write 9 Done Done Done false
208342024-01-10T18:03:44.832Z 7 Acked 1006 Write 3 Done Done Done false
208352024-01-10T18:03:44.832Z 8 Acked 1007 Write 8 Done Done Done false
208362024-01-10T18:03:44.832Z 9 Acked 1008 Write 10 Done Done Done false
208372024-01-10T18:03:44.832Z 10 Acked 1009 Write 3 Sent Done Done false
208382024-01-10T18:03:44.832Z 11 Acked 1010 Write 10 Sent Done Done false
208392024-01-10T18:03:44.832Z 12 Acked 1011 Write 10 Sent Done Done false
208402024-01-10T18:03:44.832Z 13 Acked 1012 Read 6 Sent Done Done false
208412024-01-10T18:03:44.832Z 14 Acked 1013 Read 6 Sent Done Done false
208422024-01-10T18:03:44.832Z 15 Acked 1014 Read 9 Sent Done Done false
208432024-01-10T18:03:44.832Z 16 Acked 1015 Read 9 Sent Done Done false
208442024-01-10T18:03:44.832Z 17 Acked 1016 Write 2 Sent Done Done false
208452024-01-10T18:03:44.832Z 18 Acked 1017 Read 7 Sent Done Done false
208462024-01-10T18:03:44.834Z 19 Acked 1018 Write 7 Sent Done Done false
208472024-01-10T18:03:44.834Z 20 Acked 1019 Write 2 Sent Done Done false
208482024-01-10T18:03:44.834Z 21 Acked 1020 Read 10 Sent Done Done false
208492024-01-10T18:03:44.834Z 22 Acked 1021 Read 6 Sent Done Done false
208502024-01-10T18:03:44.834Z 23 Acked 1022 Read 5 Sent Done Done false
208512024-01-10T18:03:44.834Z 24 Acked 1023 Write 3 Sent Done Done false
208522024-01-10T18:03:44.834Z 25 Acked 1024 Read 1 Sent Done Done false
208532024-01-10T18:03:44.834Z 26 Acked 1025 Read 2 Sent Done Done false
208542024-01-10T18:03:44.834Z 27 Acked 1026 Read 5 Sent Done Done false
208552024-01-10T18:03:44.834Z 28 Acked 1027 Read 4 Sent Done Done false
208562024-01-10T18:03:44.834Z 29 Acked 1028 Read 8 Sent Done Done false
208572024-01-10T18:03:44.834Z 30 Acked 1029 Read 2 Sent Sent Done false
208582024-01-10T18:03:44.834Z STATES DS:0 DS:1 DS:2 TOTAL
208592024-01-10T18:03:44.834Z New 0 0 0 0
208602024-01-10T18:03:44.834Z Sent 21 1 0 22
208612024-01-10T18:03:44.834Z Done 6 26 27 59
208622024-01-10T18:03:44.834Z Skipped 0 0 0 0
208632024-01-10T18:03:44.834Z Error 0 0 0 0
208642024-01-10T18:03:44.834ZLast Flush: 1002 1002 1002
208652024-01-10T18:03:44.834ZDownstairs last five completed: 1002 1001 1000
208662024-01-10T18:03:44.834ZUpstairs last five completed: 30 29 28 27 26
208672024-01-10T18:03:44.834ZWrote out file "/tmp/test_repair_verify.data"
208682024-01-10T18:03:44.838Z
208692024-01-10T18:03:44.841ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
208702024-01-10T18:03:44.870ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208712024-01-10T18:03:44.870Z 0 000-019 193 193 193 357 357 357 F F F
208722024-01-10T18:03:44.870Z 1 020-039 193 193 193 357 357 357 F F F
208732024-01-10T18:03:44.870Z 2 040-059 193 193 193 357 357 357 F F F
208742024-01-10T18:03:44.870Z 3 060-079 184 184 184 343 343 343 F T T <---
208752024-01-10T18:03:44.870Z 4 080-099 191 191 191 354 354 354 T T T
208762024-01-10T18:03:44.876Z 5 100-119 188 188 188 348 348 348 T T T
208772024-01-10T18:03:44.877Z 6 120-139 184 184 184 343 343 343 F F F
208782024-01-10T18:03:44.877Z 7 140-159 193 193 193 357 357 357 F T T <---
208792024-01-10T18:03:44.877Z 8 160-179 193 193 193 357 357 357 F F F
208802024-01-10T18:03:44.877Z 9 180-199 191 191 191 354 354 354 F F F
208812024-01-10T18:03:44.877Z 10 200-219 193 193 193 357 357 357 F F F
208822024-01-10T18:03:44.877Z 11 220-239 189 189 189 351 351 351 F F F
208832024-01-10T18:03:44.877Z 12 240-259 193 193 193 357 357 357 F F F
208842024-01-10T18:03:44.877Z 13 260-279 191 191 191 354 354 354 F F F
208852024-01-10T18:03:44.878Z 14 280-299 189 189 189 351 351 351 F F F
208862024-01-10T18:03:44.878Z 15 300-319 194 194 194 358 358 358 F T T <---
208872024-01-10T18:03:44.879Z 16 320-339 188 188 188 348 348 348 T T T
208882024-01-10T18:03:44.879Z 17 340-359 191 191 191 354 354 354 F F F
208892024-01-10T18:03:44.879Z 18 360-379 189 189 189 351 351 351 T T T
208902024-01-10T18:03:44.879Z 19 380-399 193 193 193 357 357 357 F F F
208912024-01-10T18:03:44.879Z 20 400-419 192 192 192 356 356 356 T T T
208922024-01-10T18:03:44.879Z 21 420-439 188 188 188 350 350 350 F F F
208932024-01-10T18:03:44.879Z 22 440-459 190 190 190 352 352 352 F F F
208942024-01-10T18:03:44.880Z 23 460-479 191 191 191 354 354 354 F F F
208952024-01-10T18:03:44.881Z 24 480-499 182 182 182 338 338 338 F F F
208962024-01-10T18:03:44.881Z 25 500-519 193 193 193 357 357 357 F F F
208972024-01-10T18:03:44.881Z 26 520-539 188 188 188 349 349 349 F F F
208982024-01-10T18:03:44.881Z 27 540-559 193 193 193 357 357 357 T T T
208992024-01-10T18:03:44.881Z 28 560-579 193 193 193 357 357 357 T T T
209002024-01-10T18:03:44.881Z 29 580-599 193 193 193 357 357 357 F T T <---
209012024-01-10T18:03:44.882ZMax gen: 194, Max flush: 358
209022024-01-10T18:03:44.882ZError: Difference in extent metadata found!
209032024-01-10T18:03:44.882ZA Difference in extent metadata is expected here
209042024-01-10T18:03:44.882ZOn loop 97
209052024-01-10T18:03:44.882Z
209062024-01-10T18:03:45.302ZVerifying data now
209072024-01-10T18:03:51.213ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
209082024-01-10T18:03:51.218Z325c325,329
209092024-01-10T18:03:51.218Z< 9,
209102024-01-10T18:03:51.218Z---
209112024-01-10T18:03:51.218Z> 10,
209122024-01-10T18:03:51.218Z> 13,
209132024-01-10T18:03:51.218Z> 15,
209142024-01-10T18:03:51.218Z> 15,
209152024-01-10T18:03:51.218Z> 13,
209162024-01-10T18:03:51.218Z327,329d330
209172024-01-10T18:03:51.218Z< 14,
209182024-01-10T18:03:51.218Z< 14,
209192024-01-10T18:03:51.218Z< 12,
209202024-01-10T18:03:51.219Z331c332
209212024-01-10T18:03:51.219Z< 10,
209222024-01-10T18:03:51.219Z---
209232024-01-10T18:03:51.219Z> 9,
209242024-01-10T18:03:51.219Z333,334c334
209252024-01-10T18:03:51.219Z< 7,
209262024-01-10T18:03:51.219Z< 10,
209272024-01-10T18:03:51.219Z---
209282024-01-10T18:03:51.219Z> 11,
209292024-01-10T18:03:51.219Z571a572,573
209302024-01-10T18:03:51.219Z> 15,
209312024-01-10T18:03:51.219Z> 15,
209322024-01-10T18:03:51.219Z574,576c576,577
209332024-01-10T18:03:51.219Z< 13,
209342024-01-10T18:03:51.219Z< 13,
209352024-01-10T18:03:51.220Z< 14,
209362024-01-10T18:03:51.220Z---
209372024-01-10T18:03:51.220Z> 15,
209382024-01-10T18:03:51.220Z> 18,
209392024-01-10T18:03:51.220Z579d579
209402024-01-10T18:03:51.220Z< 16,
209412024-01-10T18:03:51.220Zdiff found after verify
209422024-01-10T18:03:51.220ZLoop: 97 Downstairs dump after verify (and repair):
209432024-01-10T18:03:51.231ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209442024-01-10T18:03:51.231Z 0 000-019 193 193 193 357 357 357 F F F
209452024-01-10T18:03:51.231Z 1 020-039 193 193 193 357 357 357 F F F
209462024-01-10T18:03:51.231Z 2 040-059 193 193 193 357 357 357 F F F
209472024-01-10T18:03:51.231Z 3 060-079 195 195 195 359 359 359 F F F
209482024-01-10T18:03:51.231Z 4 080-099 195 195 195 359 359 359 F F F
209492024-01-10T18:03:51.235Z 5 100-119 195 195 195 359 359 359 F F F
209502024-01-10T18:03:51.235Z 6 120-139 184 184 184 343 343 343 F F F
209512024-01-10T18:03:51.235Z 7 140-159 195 195 195 359 359 359 F F F
209522024-01-10T18:03:51.235Z 8 160-179 193 193 193 357 357 357 F F F
209532024-01-10T18:03:51.235Z 9 180-199 191 191 191 354 354 354 F F F
209542024-01-10T18:03:51.235Z 10 200-219 193 193 193 357 357 357 F F F
209552024-01-10T18:03:51.235Z 11 220-239 189 189 189 351 351 351 F F F
209562024-01-10T18:03:51.235Z 12 240-259 193 193 193 357 357 357 F F F
209572024-01-10T18:03:51.235Z 13 260-279 191 191 191 354 354 354 F F F
209582024-01-10T18:03:51.235Z 14 280-299 189 189 189 351 351 351 F F F
209592024-01-10T18:03:51.235Z 15 300-319 195 195 195 359 359 359 F F F
209602024-01-10T18:03:51.236Z 16 320-339 195 195 195 359 359 359 F F F
209612024-01-10T18:03:51.236Z 17 340-359 191 191 191 354 354 354 F F F
209622024-01-10T18:03:51.236Z 18 360-379 195 195 195 359 359 359 F F F
209632024-01-10T18:03:51.236Z 19 380-399 193 193 193 357 357 357 F F F
209642024-01-10T18:03:51.236Z 20 400-419 195 195 195 359 359 359 F F F
209652024-01-10T18:03:51.236Z 21 420-439 188 188 188 350 350 350 F F F
209662024-01-10T18:03:51.236Z 22 440-459 190 190 190 352 352 352 F F F
209672024-01-10T18:03:51.236Z 23 460-479 191 191 191 354 354 354 F F F
209682024-01-10T18:03:51.236Z 24 480-499 182 182 182 338 338 338 F F F
209692024-01-10T18:03:51.236Z 25 500-519 193 193 193 357 357 357 F F F
209702024-01-10T18:03:51.236Z 26 520-539 188 188 188 349 349 349 F F F
209712024-01-10T18:03:51.236Z 27 540-559 195 195 195 359 359 359 F F F
209722024-01-10T18:03:51.236Z 28 560-579 195 195 195 359 359 359 F F F
209732024-01-10T18:03:51.236Z 29 580-599 195 195 195 359 359 359 F F F
209742024-01-10T18:03:51.236ZMax gen: 195, Max flush: 359
209752024-01-10T18:03:51.239Z
209762024-01-10T18:03:51.240ZBegin loop 98
209772024-01-10T18:03:51.240ZDownstairs to restart: 0
209782024-01-10T18:03:51.267Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.263801296Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945}
209792024-01-10T18:03:51.267Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.264321933Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945}
209802024-01-10T18:03:51.267Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.264389427Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945}
209812024-01-10T18:03:51.272Z{"msg":"Crucible aed98357-d5e2-4a6e-8abf-9447b397bdba has session id: 76dfb5b2-49d1-43b7-b993-b88160a8c66d","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.264429894Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209822024-01-10T18:03:51.272Z{"msg":"Upstairs opts: Upstairs UUID: aed98357-d5e2-4a6e-8abf-9447b397bdba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
209832024-01-10T18:03:51.272Z,"time":"2024-01-10T18:03:51.264488908Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209842024-01-10T18:03:51.272Z{"msg":"Crucible stats registered with UUID: aed98357-d5e2-4a6e-8abf-9447b397bdba","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.264612234Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209852024-01-10T18:03:51.272Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.264662571Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945}
209862024-01-10T18:03:51.272Z{"msg":"aed98357-d5e2-4a6e-8abf-9447b397bdba active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.264866805Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209872024-01-10T18:03:51.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.264930776Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"0","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209882024-01-10T18:03:51.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.264958007Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"0","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209892024-01-10T18:03:51.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.265015597Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"1","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209902024-01-10T18:03:51.273Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.265050124Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"1","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209912024-01-10T18:03:51.273Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.26508985Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"2","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209922024-01-10T18:03:51.273Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.265126037Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"2","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209932024-01-10T18:03:51.273Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.265149977Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"io task","client":"0","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209942024-01-10T18:03:51.273Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.265214431Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"io task","client":"2","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209952024-01-10T18:03:51.273Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.265304988Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"io task","client":"1","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209962024-01-10T18:03:51.273Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.265356816Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"io task","client":"0","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209972024-01-10T18:03:51.273Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.265403976Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"io task","client":"2","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209982024-01-10T18:03:51.273Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.265477144Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"io task","client":"1","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
209992024-01-10T18:03:51.273Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.266719288Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"1","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210002024-01-10T18:03:51.274Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.266811317Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"1","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210012024-01-10T18:03:51.274Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.266844398Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"2","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210022024-01-10T18:03:51.274Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.266872307Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"2","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210032024-01-10T18:03:51.280Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.267707962Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"2","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210042024-01-10T18:03:51.280Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.267777208Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"0","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210052024-01-10T18:03:51.280Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.267803757Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"0","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210062024-01-10T18:03:51.280Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.26796179Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"1","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210072024-01-10T18:03:51.280Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.268611202Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"2","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210082024-01-10T18:03:51.280Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.268724823Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"2","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210092024-01-10T18:03:51.280Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.268768667Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210102024-01-10T18:03:51.281Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.268962974Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"1","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210112024-01-10T18:03:51.281Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.269049332Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"1","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210122024-01-10T18:03:51.281Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.269122113Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210132024-01-10T18:03:51.281Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.269461053Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"0","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210142024-01-10T18:03:51.281Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270136089Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"0","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210152024-01-10T18:03:51.282Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270236332Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"0","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210162024-01-10T18:03:51.282Z{"msg":"[0]R flush_numbers[0..12]: [357, 357, 357, 359, 359, 359, 343, 359, 357, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270275154Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210172024-01-10T18:03:51.282Z{"msg":"[0]R generation[0..12]: [193, 193, 193, 195, 195, 195, 184, 195, 193, 191, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270321453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210182024-01-10T18:03:51.282Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270347802Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210192024-01-10T18:03:51.282Z{"msg":"[1]R flush_numbers[0..12]: [357, 357, 357, 359, 359, 359, 343, 359, 357, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270371622Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210202024-01-10T18:03:51.282Z{"msg":"[1]R generation[0..12]: [193, 193, 193, 195, 195, 195, 184, 195, 193, 191, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270412854Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210212024-01-10T18:03:51.282Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270437959Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210222024-01-10T18:03:51.282Z{"msg":"[2]R flush_numbers[0..12]: [357, 357, 357, 359, 359, 359, 343, 359, 357, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270462329Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210232024-01-10T18:03:51.283Z{"msg":"[2]R generation[0..12]: [193, 193, 193, 195, 195, 195, 184, 195, 193, 191, 193, 189]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
210242024-01-10T18:03:51.283Z,"time":"2024-01-10T18:03:51.270503918Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210252024-01-10T18:03:51.283Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270532591Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210262024-01-10T18:03:51.283Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.27055453Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210272024-01-10T18:03:51.283Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270596652Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210282024-01-10T18:03:51.283Z{"msg":"Next flush: 360","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270619827Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210292024-01-10T18:03:51.283Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270641537Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"":"downstairs","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210302024-01-10T18:03:51.285Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270680947Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210312024-01-10T18:03:51.285Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270702181Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"0","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210322024-01-10T18:03:51.285Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270724506Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"1","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210332024-01-10T18:03:51.285Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.27076907Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"client":"2","":"downstairs","session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210342024-01-10T18:03:51.285Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270792403Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210352024-01-10T18:03:51.285Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270812742Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210362024-01-10T18:03:51.285Z{"msg":"aed98357-d5e2-4a6e-8abf-9447b397bdba is now active with session: 76dfb5b2-49d1-43b7-b993-b88160a8c66d","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270859399Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210372024-01-10T18:03:51.285Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270880209Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945,"session_id":"76dfb5b2-49d1-43b7-b993-b88160a8c66d"}
210382024-01-10T18:03:51.285Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:51.270900544Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1945}
210392024-01-10T18:03:51.285ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
210402024-01-10T18:03:51.285ZLoading write count information from file "/tmp/test_repair_verify.data"
210412024-01-10T18:03:51.285ZRun Repair workload
210422024-01-10T18:03:51.287Z01/30 Flush
210432024-01-10T18:03:51.287Z02/30 Write block 270 len 8192 data: 15 14
210442024-01-10T18:03:51.290Z03/30 Read block 398 len 28672
210452024-01-10T18:03:51.290Z04/30 Write block 144 len 32768 data: 15 16 17 19 17 15 14 12
210462024-01-10T18:03:51.290Z05/30 Read block 500 len 16384
210472024-01-10T18:03:51.290Z06/30 Flush
210482024-01-10T18:03:51.290Z07/30 Write block 36 len 20480 data: 17 16 14 13 12
210492024-01-10T18:03:51.290Z08/30 Flush
210502024-01-10T18:03:51.294Z09/30 Write block 440 len 40960 data: 14 16 12 12 11 15 17 17 15 15
210512024-01-10T18:03:51.294Z10/30 Write block 174 len 20480 data: 16 13 12 15 14
210522024-01-10T18:03:51.294Z11/30 Read block 474 len 36864
210532024-01-10T18:03:51.300Z12/30 Read block 53 len 16384
210542024-01-10T18:03:51.303Z13/30 Read block 254 len 12288
210552024-01-10T18:03:51.303Z14/30 Read block 315 len 4096
210562024-01-10T18:03:51.303Z15/30 Read block 567 len 12288
210572024-01-10T18:03:51.303Z16/30 Write block 48 len 32768 data: 14 16 18 15 14 11 17 15
210582024-01-10T18:03:51.306Z17/30 Write block 290 len 12288 data: 17 18 19
210592024-01-10T18:03:51.306Z18/30 Read block 537 len 40960
210602024-01-10T18:03:51.309Z19/30 Write block 573 len 8192 data: 15 18
210612024-01-10T18:03:51.312Z20/30 Write block 344 len 36864 data: 16 14 15 16 12 12 11 12 15
210622024-01-10T18:03:51.312Z21/30 Read block 289 len 12288
210632024-01-10T18:03:51.315Z22/30 Write block 503 len 32768 data: 14 11 12 14 14 10 19 18
210642024-01-10T18:03:51.315Z23/30 Read block 43 len 32768
210652024-01-10T18:03:51.318Z24/30 Write block 332 len 40960 data: 9 7 7 15 14 14 14 17 13 14
210662024-01-10T18:03:51.321Z25/30 Flush
210672024-01-10T18:03:51.323Z26/30 Write block 545 len 32768 data: 17 16 18 19 20 22 22 25
210682024-01-10T18:03:51.327Z27/30 Write block 589 len 32768 data: 12 12 13 12 12 15 11 9
210692024-01-10T18:03:51.328Z28/30 Write block 4 len 32768 data: 11 11 11 10 10 13 15 16
210702024-01-10T18:03:51.328Z29/30 Read block 311 len 4096
210712024-01-10T18:03:51.331Z30/30 Write block 468 len 24576 data: 17 14 13 11 11 11
210722024-01-10T18:03:51.332Z----------------------------------------------------------------
210732024-01-10T18:03:51.332Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:29
210742024-01-10T18:03:51.332ZStates: Active Active Active
210752024-01-10T18:03:51.332ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
210762024-01-10T18:03:51.332Z 2 Acked 1001 Write 2 Sent Done Done false
210772024-01-10T18:03:51.332Z 3 Acked 1002 Read 7 Sent Done Done false
210782024-01-10T18:03:51.332Z 4 Acked 1003 Write 8 Sent Done Done false
210792024-01-10T18:03:51.332Z 5 Acked 1004 Read 4 Sent Done Done false
210802024-01-10T18:03:51.332Z 6 Acked 1005 Flush 0 Sent Done Done false
210812024-01-10T18:03:51.332Z 7 Acked 1006 Write 5 Sent Done Done false
210822024-01-10T18:03:51.332Z 8 Acked 1007 Flush 0 Sent Done Done false
210832024-01-10T18:03:51.332Z 9 Acked 1008 Write 10 Sent Done Done false
210842024-01-10T18:03:51.332Z 10 Acked 1009 Write 5 Sent Done Done false
210852024-01-10T18:03:51.332Z 11 Acked 1010 Read 9 Sent Done Done false
210862024-01-10T18:03:51.332Z 12 Acked 1011 Read 4 Sent Done Done false
210872024-01-10T18:03:51.332Z 13 Acked 1012 Read 3 Sent Done Done false
210882024-01-10T18:03:51.332Z 14 Acked 1013 Read 1 Sent Done Done false
210892024-01-10T18:03:51.332Z 15 Acked 1014 Read 3 Sent Done Done false
210902024-01-10T18:03:51.332Z 16 Acked 1015 Write 8 Sent Done Done false
210912024-01-10T18:03:51.333Z 17 Acked 1016 Write 3 Sent Done Done false
210922024-01-10T18:03:51.333Z 18 Acked 1017 Read 10 Sent Done Done false
210932024-01-10T18:03:51.333Z 19 Acked 1018 Write 2 Sent Done Done false
210942024-01-10T18:03:51.333Z 20 Acked 1019 Write 9 Sent Done Done false
210952024-01-10T18:03:51.333Z 21 Acked 1020 Read 3 Sent Done Done false
210962024-01-10T18:03:51.333Z 22 Acked 1021 Write 8 Sent Done Done false
210972024-01-10T18:03:51.333Z 23 Acked 1022 Read 8 Sent Done Done false
210982024-01-10T18:03:51.333Z 24 Acked 1023 Write 10 Sent Done Done false
210992024-01-10T18:03:51.334Z 25 Acked 1024 Flush 0 Sent Done Done false
211002024-01-10T18:03:51.334Z 26 Acked 1025 Write 8 Sent Done Done false
211012024-01-10T18:03:51.334Z 27 Acked 1026 Write 8 Sent Done Done false
211022024-01-10T18:03:51.334Z 28 Acked 1027 Write 8 Sent Done Done false
211032024-01-10T18:03:51.334Z 29 Acked 1028 Read 1 Sent Done Done false
211042024-01-10T18:03:51.334Z 30 Acked 1029 Write 6 Sent Sent Sent false
211052024-01-10T18:03:51.334Z STATES DS:0 DS:1 DS:2 TOTAL
211062024-01-10T18:03:51.334Z New 0 0 0 0
211072024-01-10T18:03:51.334Z Sent 29 1 1 31
211082024-01-10T18:03:51.334Z Done 0 28 28 56
211092024-01-10T18:03:51.334Z Skipped 0 0 0 0
211102024-01-10T18:03:51.334Z Error 0 0 0 0
211112024-01-10T18:03:51.334ZLast Flush: 1000 1024 1024
211122024-01-10T18:03:51.334ZDownstairs last five completed: 1000
211132024-01-10T18:03:51.334ZUpstairs last five completed: 30 29 28 27 26
211142024-01-10T18:03:51.334ZWrote out file "/tmp/test_repair_verify.data"
211152024-01-10T18:03:51.338Z
211162024-01-10T18:03:51.341ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211172024-01-10T18:03:51.364ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211182024-01-10T18:03:51.364Z 0 000-019 193 193 193 357 357 357 F T T <---
211192024-01-10T18:03:51.364Z 1 020-039 193 196 196 357 362 362 F F F <---
211202024-01-10T18:03:51.364Z 2 040-059 193 196 196 357 363 363 F F F <---
211212024-01-10T18:03:51.364Z 3 060-079 195 195 195 359 359 359 F F F
211222024-01-10T18:03:51.364Z 4 080-099 195 195 195 359 359 359 F F F
211232024-01-10T18:03:51.364Z 5 100-119 195 195 195 359 359 359 F F F
211242024-01-10T18:03:51.364Z 6 120-139 184 184 184 343 343 343 F F F
211252024-01-10T18:03:51.367Z 7 140-159 195 196 196 359 361 361 F F F <---
211262024-01-10T18:03:51.367Z 8 160-179 193 196 196 357 363 363 F F F <---
211272024-01-10T18:03:51.367Z 9 180-199 191 191 191 354 354 354 F F F
211282024-01-10T18:03:51.367Z 10 200-219 193 193 193 357 357 357 F F F
211292024-01-10T18:03:51.367Z 11 220-239 189 189 189 351 351 351 F F F
211302024-01-10T18:03:51.367Z 12 240-259 193 193 193 357 357 357 F F F
211312024-01-10T18:03:51.367Z 13 260-279 191 196 196 354 361 361 F F F <---
211322024-01-10T18:03:51.367Z 14 280-299 189 196 196 351 363 363 F F F <---
211332024-01-10T18:03:51.368Z 15 300-319 195 195 195 359 359 359 F F F
211342024-01-10T18:03:51.368Z 16 320-339 195 196 196 359 363 363 F F F <---
211352024-01-10T18:03:51.368Z 17 340-359 191 196 196 354 363 363 F F F <---
211362024-01-10T18:03:51.368Z 18 360-379 195 195 195 359 359 359 F F F
211372024-01-10T18:03:51.368Z 19 380-399 193 193 193 357 357 357 F F F
211382024-01-10T18:03:51.368Z 20 400-419 195 195 195 359 359 359 F F F
211392024-01-10T18:03:51.369Z 21 420-439 188 188 188 350 350 350 F F F
211402024-01-10T18:03:51.369Z 22 440-459 190 196 196 352 363 363 F F F <---
211412024-01-10T18:03:51.369Z 23 460-479 191 191 191 354 354 354 F T T <---
211422024-01-10T18:03:51.369Z 24 480-499 182 182 182 338 338 338 F F F
211432024-01-10T18:03:51.369Z 25 500-519 193 196 196 357 363 363 F F F <---
211442024-01-10T18:03:51.369Z 26 520-539 188 188 188 349 349 349 F F F
211452024-01-10T18:03:51.369Z 27 540-559 195 195 195 359 359 359 F T T <---
211462024-01-10T18:03:51.369Z 28 560-579 195 196 196 359 363 363 F F F <---
211472024-01-10T18:03:51.369Z 29 580-599 195 195 195 359 359 359 F T T <---
211482024-01-10T18:03:51.369ZMax gen: 196, Max flush: 363
211492024-01-10T18:03:51.369ZError: Difference in extent metadata found!
211502024-01-10T18:03:51.373ZA Difference in extent metadata is expected here
211512024-01-10T18:03:51.373ZOn loop 98
211522024-01-10T18:03:51.373Z
211532024-01-10T18:03:51.373ZVerifying data now
211542024-01-10T18:03:56.782ZNo change after verify
211552024-01-10T18:03:56.783ZLoop: 98 Downstairs dump after verify (and repair):
211562024-01-10T18:03:56.805ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211572024-01-10T18:03:56.805Z 0 000-019 197 197 197 364 364 364 F F F
211582024-01-10T18:03:56.805Z 1 020-039 196 196 196 362 362 362 F F F
211592024-01-10T18:03:56.805Z 2 040-059 196 196 196 363 363 363 F F F
211602024-01-10T18:03:56.805Z 3 060-079 195 195 195 359 359 359 F F F
211612024-01-10T18:03:56.805Z 4 080-099 195 195 195 359 359 359 F F F
211622024-01-10T18:03:56.805Z 5 100-119 195 195 195 359 359 359 F F F
211632024-01-10T18:03:56.805Z 6 120-139 184 184 184 343 343 343 F F F
211642024-01-10T18:03:56.808Z 7 140-159 196 196 196 361 361 361 F F F
211652024-01-10T18:03:56.808Z 8 160-179 196 196 196 363 363 363 F F F
211662024-01-10T18:03:56.813Z 9 180-199 191 191 191 354 354 354 F F F
211672024-01-10T18:03:56.814Z 10 200-219 193 193 193 357 357 357 F F F
211682024-01-10T18:03:56.814Z 11 220-239 189 189 189 351 351 351 F F F
211692024-01-10T18:03:56.814Z 12 240-259 193 193 193 357 357 357 F F F
211702024-01-10T18:03:56.814Z 13 260-279 196 196 196 361 361 361 F F F
211712024-01-10T18:03:56.814Z 14 280-299 196 196 196 363 363 363 F F F
211722024-01-10T18:03:56.814Z 15 300-319 195 195 195 359 359 359 F F F
211732024-01-10T18:03:56.814Z 16 320-339 196 196 196 363 363 363 F F F
211742024-01-10T18:03:56.814Z 17 340-359 196 196 196 363 363 363 F F F
211752024-01-10T18:03:56.814Z 18 360-379 195 195 195 359 359 359 F F F
211762024-01-10T18:03:56.814Z 19 380-399 193 193 193 357 357 357 F F F
211772024-01-10T18:03:56.814Z 20 400-419 195 195 195 359 359 359 F F F
211782024-01-10T18:03:56.814Z 21 420-439 188 188 188 350 350 350 F F F
211792024-01-10T18:03:56.815Z 22 440-459 196 196 196 363 363 363 F F F
211802024-01-10T18:03:56.816Z 23 460-479 197 197 197 364 364 364 F F F
211812024-01-10T18:03:56.816Z 24 480-499 182 182 182 338 338 338 F F F
211822024-01-10T18:03:56.816Z 25 500-519 196 196 196 363 363 363 F F F
211832024-01-10T18:03:56.816Z 26 520-539 188 188 188 349 349 349 F F F
211842024-01-10T18:03:56.816Z 27 540-559 197 197 197 364 364 364 F F F
211852024-01-10T18:03:56.816Z 28 560-579 196 196 196 363 363 363 F F F
211862024-01-10T18:03:56.817Z 29 580-599 197 197 197 364 364 364 F F F
211872024-01-10T18:03:56.817ZMax gen: 197, Max flush: 364
211882024-01-10T18:03:56.817Z
211892024-01-10T18:03:56.817ZBegin loop 99
211902024-01-10T18:03:56.817ZDownstairs to restart: 1
211912024-01-10T18:03:56.838Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.834546955Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953}
211922024-01-10T18:03:56.838Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"99c957c6a74e6d7d6180d15bada5890df4de8432\",\n git_commit_timestamp: \"2024-01-10T17:30:23.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.835079492Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953}
211932024-01-10T18:03:56.843Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.835170574Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953}
211942024-01-10T18:03:56.843Z{"msg":"Crucible 3018cf6a-3162-4537-9b1f-4d6fe26ab351 has session id: 160ce4de-ba9c-42ec-b727-43897faade68","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.835239787Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
211952024-01-10T18:03:56.843Z{"msg":"Upstairs opts: Upstairs UUID: 3018cf6a-3162-4537-9b1f-4d6fe26ab351, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
211962024-01-10T18:03:56.843Z,"time":"2024-01-10T18:03:56.835347309Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
211972024-01-10T18:03:56.846Z{"msg":"Crucible stats registered with UUID: 3018cf6a-3162-4537-9b1f-4d6fe26ab351","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.83539101Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
211982024-01-10T18:03:56.846Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.83550889Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953}
211992024-01-10T18:03:56.846Z{"msg":"3018cf6a-3162-4537-9b1f-4d6fe26ab351 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.835719737Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212002024-01-10T18:03:56.846Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.835842658Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"0","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212012024-01-10T18:03:56.846Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.835880281Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"0","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212022024-01-10T18:03:56.846Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.83597813Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"1","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212032024-01-10T18:03:56.847Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.836035069Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"1","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212042024-01-10T18:03:56.847Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.836063871Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"2","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212052024-01-10T18:03:56.847Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.836135744Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"2","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212062024-01-10T18:03:56.847Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.836190126Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"io task","client":"0","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212072024-01-10T18:03:56.847Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.836222312Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"io task","client":"2","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212082024-01-10T18:03:56.849Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.836417212Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"io task","client":"1","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212092024-01-10T18:03:56.849Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.836508724Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"io task","client":"0","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212102024-01-10T18:03:56.849Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.836549799Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"io task","client":"2","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212112024-01-10T18:03:56.849Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.836642285Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"io task","client":"1","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212122024-01-10T18:03:56.849Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.837464185Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"2","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212132024-01-10T18:03:56.849Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.837553727Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"2","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212142024-01-10T18:03:56.849Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.837603141Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"0","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212152024-01-10T18:03:56.849Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.837701748Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"0","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212162024-01-10T18:03:56.850Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.838297829Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"2","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212172024-01-10T18:03:56.850Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.838537023Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"1","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212182024-01-10T18:03:56.850Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.838620186Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"1","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212192024-01-10T18:03:56.850Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.838647247Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"0","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212202024-01-10T18:03:56.850Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.83909242Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"2","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212212024-01-10T18:03:56.851Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.839164281Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"2","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212222024-01-10T18:03:56.856Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.839211556Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212232024-01-10T18:03:56.856Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.839486048Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"0","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212242024-01-10T18:03:56.856Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.839546125Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"0","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212252024-01-10T18:03:56.856Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.839571629Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212262024-01-10T18:03:56.856Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.83961251Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"1","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212272024-01-10T18:03:56.856Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840293554Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"1","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212282024-01-10T18:03:56.857Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840377064Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"1","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212292024-01-10T18:03:56.857Z{"msg":"[0]R flush_numbers[0..12]: [364, 362, 363, 359, 359, 359, 343, 361, 363, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840406636Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212302024-01-10T18:03:56.857Z{"msg":"[0]R generation[0..12]: [197, 196, 196, 195, 195, 195, 184, 196, 196, 191, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.84044958Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212312024-01-10T18:03:56.857Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840475512Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212322024-01-10T18:03:56.857Z{"msg":"[1]R flush_numbers[0..12]: [364, 362, 363, 359, 359, 359, 343, 361, 363, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840517752Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212332024-01-10T18:03:56.857Z{"msg":"[1]R generation[0..12]: [197, 196, 196, 195, 195, 195, 184, 196, 196, 191, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840542022Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212342024-01-10T18:03:56.858Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
212352024-01-10T18:03:56.858Z,"time":"2024-01-10T18:03:56.840602282Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212362024-01-10T18:03:56.858Z{"msg":"[2]R flush_numbers[0..12]: [364, 362, 363, 359, 359, 359, 343, 361, 363, 354, 357, 351]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840629747Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212372024-01-10T18:03:56.858Z{"msg":"[2]R generation[0..12]: [197, 196, 196, 195, 195, 195, 184, 196, 196, 191, 193, 189]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840672472Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212382024-01-10T18:03:56.859Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840697603Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212392024-01-10T18:03:56.859Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840737534Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212402024-01-10T18:03:56.859Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840760635Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212412024-01-10T18:03:56.859Z{"msg":"Next flush: 365","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840807096Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212422024-01-10T18:03:56.859Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.84084365Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"":"downstairs","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212432024-01-10T18:03:56.859Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840893318Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212442024-01-10T18:03:56.859Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840914632Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"0","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212452024-01-10T18:03:56.859Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840961197Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"1","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212462024-01-10T18:03:56.859Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.840983972Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"client":"2","":"downstairs","session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212472024-01-10T18:03:56.859Z{"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.84103102Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212482024-01-10T18:03:56.860ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
212492024-01-10T18:03:56.860Z{"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.841055896Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212502024-01-10T18:03:56.860Z{"msg":"3018cf6a-3162-4537-9b1f-4d6fe26ab351 is now active with session: 160ce4de-ba9c-42ec-b727-43897faade68","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.841095765Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212512024-01-10T18:03:56.861Z{"msg":"Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.841116399Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953,"session_id":"160ce4de-ba9c-42ec-b727-43897faade68"}
212522024-01-10T18:03:56.861Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-01-10T18:03:56.841155088Z","hostname":"ip-10-150-1-7.us-west-2.compute.internal","pid":1953}
212532024-01-10T18:03:56.861ZLoading write count information from file "/tmp/test_repair_verify.data"
212542024-01-10T18:03:56.861ZRun Repair workload
212552024-01-10T18:03:56.861Z01/30 Flush
212562024-01-10T18:03:56.861Z02/30 Write block 375 len 32768 data: 14 12 12 9 9 10 11 11
212572024-01-10T18:03:56.862Z03/30 Write block 487 len 32768 data: 12 13 14 15 14 18 18 18
212582024-01-10T18:03:56.862Z04/30 Read block 322 len 12288
212592024-01-10T18:03:56.862Z05/30 Flush
212602024-01-10T18:03:56.877Z06/30 Write block 452 len 28672 data: 16 19 16 14 15 16 16
212612024-01-10T18:03:56.878Z07/30 Write block 352 len 16384 data: 16 12 13 11
212622024-01-10T18:03:56.878Z08/30 Write block 7 len 8192 data: 11 11
212632024-01-10T18:03:56.878Z09/30 Read block 329 len 40960
212642024-01-10T18:03:56.878Z10/30 Read block 207 len 20480
212652024-01-10T18:03:56.878Z11/30 Write block 252 len 16384 data: 14 14 12 12
212662024-01-10T18:03:56.878Z12/30 Read block 353 len 20480
212672024-01-10T18:03:56.882Z13/30 Read block 198 len 28672
212682024-01-10T18:03:56.882Z14/30 Read block 508 len 16384
212692024-01-10T18:03:56.882Z15/30 Write block 89 len 40960 data: 10 11 15 15 15 14 17 18 19 14
212702024-01-10T18:03:56.882Z16/30 Read block 252 len 8192
212712024-01-10T18:03:56.882Z17/30 Write block 355 len 20480 data: 12 13 14 16 14
212722024-01-10T18:03:56.882Z18/30 Read block 477 len 28672
212732024-01-10T18:03:56.882Z19/30 Write block 557 len 20480 data: 16 13 13 14 16
212742024-01-10T18:03:56.882Z20/30 Write block 505 len 12288 data: 13 15 15
212752024-01-10T18:03:56.882Z21/30 Write block 356 len 24576 data: 14 15 17 15 11 11
212762024-01-10T18:03:56.883Z22/30 Read block 212 len 12288
212772024-01-10T18:03:56.886Z23/30 Read block 556 len 36864
212782024-01-10T18:03:56.889Z24/30 Read block 546 len 8192
212792024-01-10T18:03:56.889Z25/30 Read block 27 len 16384
212802024-01-10T18:03:56.889Z26/30 Read block 317 len 32768
212812024-01-10T18:03:56.892Z27/30 Write block 174 len 32768 data: 17 14 13 16 15 14 16 15
212822024-01-10T18:03:56.896Z28/30 Write block 90 len 32768 data: 12 16 16 16 15 18 19 20
212832024-01-10T18:03:56.896Z29/30 Write block 248 len 4096 data: 15
212842024-01-10T18:03:56.896Z30/30 Read block 19 len 12288
212852024-01-10T18:03:56.901Z----------------------------------------------------------------
212862024-01-10T18:03:56.901Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:29
212872024-01-10T18:03:56.901ZStates: Active Active Active
212882024-01-10T18:03:56.901ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
212892024-01-10T18:03:56.901Z 2 Acked 1001 Write 8 Done Sent Done false
212902024-01-10T18:03:56.901Z 3 Acked 1002 Write 8 Done Sent Done false
212912024-01-10T18:03:56.901Z 4 Acked 1003 Read 3 Done Sent Done false
212922024-01-10T18:03:56.902Z 5 Acked 1004 Flush 0 Done Sent Done false
212932024-01-10T18:03:56.902Z 6 Acked 1005 Write 7 Done Sent Done false
212942024-01-10T18:03:56.902Z 7 Acked 1006 Write 4 Done Sent Done false
212952024-01-10T18:03:56.903Z 8 Acked 1007 Write 2 Done Sent Done false
212962024-01-10T18:03:56.903Z 9 Acked 1008 Read 10 Done Sent Done false
212972024-01-10T18:03:56.903Z 10 Acked 1009 Read 5 Done Sent Done false
212982024-01-10T18:03:56.903Z 11 Acked 1010 Write 4 Done Sent Done false
212992024-01-10T18:03:56.903Z 12 Acked 1011 Read 5 Done Sent Done false
213002024-01-10T18:03:56.903Z 13 Acked 1012 Read 7 Done Sent Done false
213012024-01-10T18:03:56.903Z 14 Acked 1013 Read 4 Done Sent Done false
213022024-01-10T18:03:56.903Z 15 Acked 1014 Write 10 Done Sent Done false
213032024-01-10T18:03:56.903Z 16 Acked 1015 Read 2 Done Sent Done false
213042024-01-10T18:03:56.903Z 17 Acked 1016 Write 5 Done Sent Done false
213052024-01-10T18:03:56.903Z 18 Acked 1017 Read 7 Done Sent Done false
213062024-01-10T18:03:56.903Z 19 Acked 1018 Write 5 Done Sent Done false
213072024-01-10T18:03:56.903Z 20 Acked 1019 Write 3 Done Sent Done false
213082024-01-10T18:03:56.903Z 21 Acked 1020 Write 6 Done Sent Done false
213092024-01-10T18:03:56.903Z 22 Acked 1021 Read 3 Done Sent Done false
213102024-01-10T18:03:56.903Z 23 Acked 1022 Read 9 Done Sent Done false
213112024-01-10T18:03:56.903Z 24 Acked 1023 Read 2 Done Sent Done false
213122024-01-10T18:03:56.903Z 25 Acked 1024 Read 4 Done Sent Done false
213132024-01-10T18:03:56.903Z 26 Acked 1025 Read 8 Done Sent Done false
213142024-01-10T18:03:56.903Z 27 Acked 1026 Write 8 Done Sent Done false
213152024-01-10T18:03:56.904Z 28 Acked 1027 Write 8 Done Sent Done false
213162024-01-10T18:03:56.904Z 29 Acked 1028 Write 1 Done Sent Sent false
213172024-01-10T18:03:56.904Z 30 Acked 1029 Read 3 Done Sent Sent false
213182024-01-10T18:03:56.904Z STATES DS:0 DS:1 DS:2 TOTAL
213192024-01-10T18:03:56.904Z New 0 0 0 0
213202024-01-10T18:03:56.904Z Sent 0 29 2 31
213212024-01-10T18:03:56.904Z Done 29 0 27 56
213222024-01-10T18:03:56.905Z Skipped 0 0 0 0
213232024-01-10T18:03:56.905Z Error 0 0 0 0
213242024-01-10T18:03:56.905ZLast Flush: 1004 1000 1004
213252024-01-10T18:03:56.905ZDownstairs last five completed: 1000
213262024-01-10T18:03:56.905ZUpstairs last five completed: 30 29 28 27 26
213272024-01-10T18:03:56.905ZWrote out file "/tmp/test_repair_verify.data"
213282024-01-10T18:03:56.908Z
213292024-01-10T18:03:56.911ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
213302024-01-10T18:03:56.939ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213312024-01-10T18:03:56.940Z 0 000-019 197 197 197 364 364 364 T F T <---
213322024-01-10T18:03:56.940Z 1 020-039 196 196 196 362 362 362 F F F
213332024-01-10T18:03:56.940Z 2 040-059 196 196 196 363 363 363 F F F
213342024-01-10T18:03:56.940Z 3 060-079 195 195 195 359 359 359 F F F
213352024-01-10T18:03:56.940Z 4 080-099 195 195 195 359 359 359 T F T <---
213362024-01-10T18:03:56.940Z 5 100-119 195 195 195 359 359 359 F F F
213372024-01-10T18:03:56.940Z 6 120-139 184 184 184 343 343 343 F F F
213382024-01-10T18:03:56.940Z 7 140-159 196 196 196 361 361 361 F F F
213392024-01-10T18:03:56.940Z 8 160-179 196 196 196 363 363 363 T F T <---
213402024-01-10T18:03:56.941Z 9 180-199 191 191 191 354 354 354 T F T <---
213412024-01-10T18:03:56.941Z 10 200-219 193 193 193 357 357 357 F F F
213422024-01-10T18:03:56.941Z 11 220-239 189 189 189 351 351 351 F F F
213432024-01-10T18:03:56.941Z 12 240-259 193 193 193 357 357 357 T F T <---
213442024-01-10T18:03:56.945Z 13 260-279 196 196 196 361 361 361 F F F
213452024-01-10T18:03:56.945Z 14 280-299 196 196 196 363 363 363 F F F
213462024-01-10T18:03:56.945Z 15 300-319 195 195 195 359 359 359 F F F
213472024-01-10T18:03:56.945Z 16 320-339 196 196 196 363 363 363 F F F
213482024-01-10T18:03:56.946Z 17 340-359 196 196 196 363 363 363 T F T <---
213492024-01-10T18:03:56.946Z 18 360-379 198 195 198 366 359 366 T F T <---
213502024-01-10T18:03:56.946Z 19 380-399 198 193 198 366 357 366 F F F <---
213512024-01-10T18:03:56.946Z 20 400-419 195 195 195 359 359 359 F F F
213522024-01-10T18:03:56.946Z 21 420-439 188 188 188 350 350 350 F F F
213532024-01-10T18:03:56.946Z 22 440-459 196 196 196 363 363 363 T F T <---
213542024-01-10T18:03:56.946Z 23 460-479 197 197 197 364 364 364 F F F
213552024-01-10T18:03:56.946Z 24 480-499 198 182 198 366 338 366 F F F <---
213562024-01-10T18:03:56.946Z 25 500-519 196 196 196 363 363 363 T F T <---
213572024-01-10T18:03:56.946Z 26 520-539 188 188 188 349 349 349 F F F
213582024-01-10T18:03:56.946Z 27 540-559 197 197 197 364 364 364 T F T <---
213592024-01-10T18:03:56.946Z 28 560-579 196 196 196 363 363 363 T F T <---
213602024-01-10T18:03:56.946Z 29 580-599 197 197 197 364 364 364 F F F
213612024-01-10T18:03:56.946ZMax gen: 198, Max flush: 366
213622024-01-10T18:03:56.947ZError: Difference in extent metadata found!
213632024-01-10T18:03:56.947ZA Difference in extent metadata is expected here
213642024-01-10T18:03:56.948ZOn loop 99
213652024-01-10T18:03:56.948Z
213662024-01-10T18:03:56.952ZVerifying data now
213672024-01-10T18:04:02.242ZNo change after verify
213682024-01-10T18:04:02.243ZLoop: 99 Downstairs dump after verify (and repair):
213692024-01-10T18:04:02.265ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213702024-01-10T18:04:02.265Z 0 000-019 199 199 199 367 367 367 F F F
213712024-01-10T18:04:02.265Z 1 020-039 196 196 196 362 362 362 F F F
213722024-01-10T18:04:02.265Z 2 040-059 196 196 196 363 363 363 F F F
213732024-01-10T18:04:02.265Z 3 060-079 195 195 195 359 359 359 F F F
213742024-01-10T18:04:02.265Z 4 080-099 199 199 199 367 367 367 F F F
213752024-01-10T18:04:02.265Z 5 100-119 195 195 195 359 359 359 F F F
213762024-01-10T18:04:02.270Z 6 120-139 184 184 184 343 343 343 F F F
213772024-01-10T18:04:02.270Z 7 140-159 196 196 196 361 361 361 F F F
213782024-01-10T18:04:02.270Z 8 160-179 199 199 199 367 367 367 F F F
213792024-01-10T18:04:02.271Z 9 180-199 199 199 199 367 367 367 F F F
213802024-01-10T18:04:02.271Z 10 200-219 193 193 193 357 357 357 F F F
213812024-01-10T18:04:02.271Z 11 220-239 189 189 189 351 351 351 F F F
213822024-01-10T18:04:02.271Z 12 240-259 199 199 199 367 367 367 F F F
213832024-01-10T18:04:02.271Z 13 260-279 196 196 196 361 361 361 F F F
213842024-01-10T18:04:02.271Z 14 280-299 196 196 196 363 363 363 F F F
213852024-01-10T18:04:02.271Z 15 300-319 195 195 195 359 359 359 F F F
213862024-01-10T18:04:02.271Z 16 320-339 196 196 196 363 363 363 F F F
213872024-01-10T18:04:02.271Z 17 340-359 199 199 199 367 367 367 F F F
213882024-01-10T18:04:02.271Z 18 360-379 199 199 199 367 367 367 F F F
213892024-01-10T18:04:02.271Z 19 380-399 198 198 198 366 366 366 F F F
213902024-01-10T18:04:02.271Z 20 400-419 195 195 195 359 359 359 F F F
213912024-01-10T18:04:02.271Z 21 420-439 188 188 188 350 350 350 F F F
213922024-01-10T18:04:02.271Z 22 440-459 199 199 199 367 367 367 F F F
213932024-01-10T18:04:02.271Z 23 460-479 197 197 197 364 364 364 F F F
213942024-01-10T18:04:02.271Z 24 480-499 198 198 198 366 366 366 F F F
213952024-01-10T18:04:02.271Z 25 500-519 199 199 199 367 367 367 F F F
213962024-01-10T18:04:02.271Z 26 520-539 188 188 188 349 349 349 F F F
213972024-01-10T18:04:02.271Z 27 540-559 199 199 199 367 367 367 F F F
213982024-01-10T18:04:02.271Z 28 560-579 199 199 199 367 367 367 F F F
213992024-01-10T18:04:02.271Z 29 580-599 197 197 197 364 364 364 F F F
214002024-01-10T18:04:02.271ZMax gen: 199, Max flush: 367
214012024-01-10T18:04:02.271Z9:03 Test duration
214022024-01-10T18:04:02.272ZTest completed
214032024-01-10T18:04:02.275Z
214042024-01-10T18:04:02.275Zreal 9:02.875333729
214052024-01-10T18:04:02.275Zuser 3:01.023139011
214062024-01-10T18:04:02.275Zsys 26.546525576
214072024-01-10T18:04:02.275Ztrap 0.131198974
214082024-01-10T18:04:02.275Ztflt 0.055958196
214092024-01-10T18:04:02.275Zdflt 0.088438624
214102024-01-10T18:04:02.275Zkflt 0.003166675
214112024-01-10T18:04:02.275Zlock 6:13:29.218554976
214122024-01-10T18:04:02.275Zslp 42:12.056877293
214132024-01-10T18:04:02.275Zlat 9.377673879
214142024-01-10T18:04:02.275Zstop 0.063910699
214152024-01-10T18:04:02.276Zprocess exited: duration 555383 ms, exit code 0
 
214162024-01-10T18:04:02.283Zfound 4 output files
214172024-01-10T18:04:02.283Zuploading: /tmp/test_repair_ds8810.txt (51747 bytes)
214182024-01-10T18:04:03.307Zuploaded: /tmp/test_repair_ds8810.txt
214192024-01-10T18:04:03.307Zuploading: /tmp/test_repair_ds8820.txt (20721 bytes)
214202024-01-10T18:04:04.327Zuploaded: /tmp/test_repair_ds8820.txt
214212024-01-10T18:04:04.327Zuploading: /tmp/test_repair_ds8830.txt (114337 bytes)
214222024-01-10T18:04:05.353Zuploaded: /tmp/test_repair_ds8830.txt
214232024-01-10T18:04:05.353Zuploading: /tmp/test_repair_out.txt (86753 bytes)
214242024-01-10T18:04:06.375Zuploaded: /tmp/test_repair_out.txt